Repository: TypNull/Tubifarry Branch: master Commit: f6ed443d1819 Files: 282 Total size: 1.6 MB Directory structure: gitextract_31q9a7rf/ ├── .gitattributes ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ ├── documentation_improvement.yml │ │ └── feature_request.yml │ └── workflows/ │ ├── branch-detection.yml │ ├── compilation.yml │ ├── dotnet-setup.yml │ ├── git-info.yml │ ├── main.yml │ ├── metadata.yml │ ├── packaging.yml │ ├── publishing.yml │ └── release-notes.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTION.md ├── Directory.Build.props ├── LICENSE.txt ├── NuGet.config ├── README.md ├── Tubifarry/ │ ├── Blocklisting/ │ │ ├── BaseBlocklist.cs │ │ └── Blocklists.cs │ ├── Core/ │ │ ├── Model/ │ │ │ ├── AlbumData.cs │ │ │ ├── ApiCircuitBreaker.cs │ │ │ ├── AudioMetadataHandler.cs │ │ │ ├── FileCache.cs │ │ │ ├── PlaylistItem.cs │ │ │ └── TrustedSessionException.cs │ │ ├── Records/ │ │ │ ├── Lyric.cs │ │ │ ├── MappingAgent.cs │ │ │ ├── MusicBrainzData.cs │ │ │ └── YouTubeSession.cs │ │ ├── Replacements/ │ │ │ ├── ExtendedHttpIndexerBase.cs │ │ │ ├── FlexibleHttpDispatcher.cs │ │ │ └── IIndexerRequestGenerator.cs │ │ ├── Telemetry/ │ │ │ ├── ISearchContextBuffer.cs │ │ │ ├── ISentryHelper.cs │ │ │ ├── NoopSentryHelper.cs │ │ │ ├── SearchContextBuffer.cs │ │ │ ├── SentryEventFilter.cs │ │ │ ├── SentryHelper.cs │ │ │ ├── SlskdBufferedContext.cs │ │ │ ├── SlskdSentryEvents.cs │ │ │ ├── SlskdTrackingService.cs │ │ │ ├── TubifarrySentry.cs │ │ │ └── TubifarrySentryTarget.cs │ │ └── Utilities/ │ │ ├── AudioFormat.cs │ │ ├── CacheService.cs │ │ ├── CookieManager.cs │ │ ├── DynamicSchemaInjector.cs │ │ ├── DynamicStateSettings.cs │ │ ├── FileInfoParser.cs │ │ ├── IndexerParserHelper.cs │ │ ├── JsonConverters.cs │ │ ├── LazyRequestChain.cs │ │ ├── PermissionTester.cs │ │ ├── PluginSettings.cs │ │ ├── ReleaseFormatter.cs │ │ ├── RepositorySettingsResolver.cs │ │ └── UserAgentValidator.cs │ ├── Debug.targets │ ├── Download/ │ │ ├── Base/ │ │ │ ├── BaseDownloadManager.cs │ │ │ ├── BaseDownloadOptions.cs │ │ │ ├── BaseDownloadRequest.cs │ │ │ └── BaseHttpClient.cs │ │ └── Clients/ │ │ ├── DABMusic/ │ │ │ ├── DABMusicClient.cs │ │ │ ├── DABMusicDownloadManager.cs │ │ │ ├── DABMusicDownloadOptions.cs │ │ │ ├── DABMusicDownloadRequest.cs │ │ │ └── DABMusicProviderSettings.cs │ │ ├── Lucida/ │ │ │ ├── ILucidaRateLimiter.cs │ │ │ ├── LucidaClient.cs │ │ │ ├── LucidaDownloadManager.cs │ │ │ ├── LucidaDownloadOptions.cs │ │ │ ├── LucidaDownloadRequest.cs │ │ │ ├── LucidaInitiationResult.cs │ │ │ ├── LucidaMetadataExtractor.cs │ │ │ ├── LucidaProviderSettings.cs │ │ │ ├── LucidaRateLimitException.cs │ │ │ ├── LucidaRateLimiter.cs │ │ │ ├── LucidaTokenExtractor.cs │ │ │ └── LucidaWorkerState.cs │ │ ├── Soulseek/ │ │ │ ├── ISlskdApiClient.cs │ │ │ ├── ISlskdDownloadManager.cs │ │ │ ├── Models/ │ │ │ │ ├── DownloadKey.cs │ │ │ │ ├── SlskdDownloadDirectory.cs │ │ │ │ ├── SlskdDownloadFile.cs │ │ │ │ ├── SlskdDownloadItem.cs │ │ │ │ └── SlskdFileState.cs │ │ │ ├── SlskdApiClient.cs │ │ │ ├── SlskdClient.cs │ │ │ ├── SlskdDownloadManager.cs │ │ │ ├── SlskdProviderSettings.cs │ │ │ ├── SlskdRetryHandler.cs │ │ │ └── SlskdStatusResolver.cs │ │ ├── SubSonic/ │ │ │ ├── SubSonicClient.cs │ │ │ ├── SubSonicDownloadManager.cs │ │ │ ├── SubSonicDownloadOptions.cs │ │ │ ├── SubSonicDownloadRequest.cs │ │ │ └── SubSonicProviderSettings.cs │ │ ├── TripleTriple/ │ │ │ ├── TripleTripleClient.cs │ │ │ ├── TripleTripleDownloadManager.cs │ │ │ ├── TripleTripleDownloadOptions.cs │ │ │ ├── TripleTripleDownloadRequest.cs │ │ │ └── TripleTripleProviderSettings.cs │ │ └── YouTube/ │ │ ├── SponsorBlock.cs │ │ ├── TrustedSessionHelper.cs │ │ ├── YouTubeDownloadOptions.cs │ │ ├── YouTubeDownloadRequest.cs │ │ ├── YoutubeClient.cs │ │ ├── YoutubeDownloadManager.cs │ │ └── YoutubeProviderSettings.cs │ ├── ILRepack.targets │ ├── ImportLists/ │ │ ├── ArrStack/ │ │ │ ├── ArrMedia.cs │ │ │ ├── ArrSoundtrackImport.cs │ │ │ ├── ArrSoundtrackImportParser.cs │ │ │ ├── ArrSoundtrackImportSettings.cs │ │ │ └── ArrSoundtrackRequestGenerator.cs │ │ ├── LastFmRecommendation/ │ │ │ ├── LastFmRecomendRequestGenerator.cs │ │ │ ├── LastFmRecommend.cs │ │ │ ├── LastFmRecommendParser.cs │ │ │ ├── LastFmRecommendSettings.cs │ │ │ └── LastFmRecords.cs │ │ ├── ListenBrainz/ │ │ │ ├── ListenBrainzCFRecommendations/ │ │ │ │ ├── ListenBrainzCFRecommendationsImportList.cs │ │ │ │ ├── ListenBrainzCFRecommendationsParser.cs │ │ │ │ ├── ListenBrainzCFRecommendationsRequestGenerator.cs │ │ │ │ └── ListenBrainzCFRecommendationsSettings.cs │ │ │ ├── ListenBrainzCreatedForPlaylist/ │ │ │ │ ├── ListenBrainzCreatedForPlaylistImportList.cs │ │ │ │ ├── ListenBrainzCreatedForPlaylistParser.cs │ │ │ │ ├── ListenBrainzCreatedForPlaylistRequestGenerator.cs │ │ │ │ └── ListenBrainzCreatedForPlaylistSettings.cs │ │ │ ├── ListenBrainzPlaylist/ │ │ │ │ ├── ListenBrainzPlaylistImportList.cs │ │ │ │ ├── ListenBrainzPlaylistParser.cs │ │ │ │ ├── ListenBrainzPlaylistRequestGenerator.cs │ │ │ │ └── ListenBrainzPlaylistSettings.cs │ │ │ ├── ListenBrainzRecords.cs │ │ │ └── ListenBrainzUserStats/ │ │ │ ├── ListenBrainzUserStatsImportList.cs │ │ │ ├── ListenBrainzUserStatsParser.cs │ │ │ ├── ListenBrainzUserStatsRequestGenerator.cs │ │ │ └── ListenBrainzUserStatsSettings.cs │ │ └── Spotify/ │ │ ├── SpotifyUserPlaylistImport.cs │ │ └── SpotifyUserPlaylistImportSettings.cs │ ├── Indexers/ │ │ ├── DABMusic/ │ │ │ ├── DABMusicIndexer.cs │ │ │ ├── DABMusicIndexerSettings.cs │ │ │ ├── DABMusicParser.cs │ │ │ ├── DABMusicRecords.cs │ │ │ ├── DABMusicRequestGenerator.cs │ │ │ └── DABMusicSessionHelper.cs │ │ ├── DownloadProtocols.cs │ │ ├── Lucida/ │ │ │ ├── LucidaIndexer.cs │ │ │ ├── LucidaIndexerSettings.cs │ │ │ ├── LucidaRecords.cs │ │ │ ├── LucidaRequestGenerator.cs │ │ │ ├── LucidaRequestParser.cs │ │ │ └── LucidaServiceHelper.cs │ │ ├── Soulseek/ │ │ │ ├── ISlskdItemsParser.cs │ │ │ ├── Search/ │ │ │ │ ├── Core/ │ │ │ │ │ ├── ISearchStrategy.cs │ │ │ │ │ ├── QueryAnalyzer.cs │ │ │ │ │ ├── SearchContext.cs │ │ │ │ │ └── SearchPipeline.cs │ │ │ │ ├── Strategies/ │ │ │ │ │ ├── BaseSearchStrategy.cs │ │ │ │ │ ├── FallbackStrategy.cs │ │ │ │ │ ├── SpecialCaseStrategy.cs │ │ │ │ │ ├── TemplateSearchStrategy.cs │ │ │ │ │ └── VariationStrategy.cs │ │ │ │ ├── Templates/ │ │ │ │ │ └── TemplateEngine.cs │ │ │ │ └── Transformers/ │ │ │ │ ├── QueryBuilder.cs │ │ │ │ └── QueryNormalizer.cs │ │ │ ├── SlsdkRecords.cs │ │ │ ├── SlskdIndexer.cs │ │ │ ├── SlskdIndexerParser.cs │ │ │ ├── SlskdItemsParser.cs │ │ │ ├── SlskdRequestGenerator.cs │ │ │ ├── SlskdSettings.cs │ │ │ └── SlskdTextProcessor.cs │ │ ├── Spotify/ │ │ │ ├── SpotifyIndexerSettings.cs │ │ │ ├── SpotifyParser.cs │ │ │ ├── SpotifyRequestGenerator.cs │ │ │ ├── SpotifyToYouTubeEnricher.cs │ │ │ └── TubifarryIndexer.cs │ │ ├── SubSonic/ │ │ │ ├── SubSonicAuthHelper.cs │ │ │ ├── SubSonicIndexer.cs │ │ │ ├── SubSonicIndexerParser.cs │ │ │ ├── SubSonicIndexerSettings.cs │ │ │ ├── SubSonicRecords.cs │ │ │ └── SubSonicRequestGenerator.cs │ │ ├── TripleTriple/ │ │ │ ├── TripleTripleIndexer.cs │ │ │ ├── TripleTripleIndexerSettings.cs │ │ │ ├── TripleTripleParser.cs │ │ │ ├── TripleTripleRecords.cs │ │ │ └── TripleTripleRequestGenerator.cs │ │ └── YouTube/ │ │ ├── YoutubeIndexer.cs │ │ ├── YoutubeIndexerSettings.cs │ │ ├── YoutubeParser.cs │ │ └── YoutubeRequestGenerator.cs │ ├── Metadata/ │ │ ├── Converter/ │ │ │ ├── AudioConverter.cs │ │ │ ├── AudioConverterSettings.cs │ │ │ └── BitrateRules.cs │ │ ├── Lyrics/ │ │ │ ├── LyricEnhancerSettings.cs │ │ │ ├── LyricsEnhancer.cs │ │ │ ├── LyricsHelper.cs │ │ │ ├── LyricsProviders.cs │ │ │ └── TrackFileRepositoryHelper.cs │ │ ├── Proxy/ │ │ │ ├── MetadataProvider/ │ │ │ │ ├── AlbumMapper.cs │ │ │ │ ├── CustomLidarr/ │ │ │ │ │ ├── CustomLidarrMetadataProxy.cs │ │ │ │ │ ├── CustomLidarrMetadataProxySettings.cs │ │ │ │ │ ├── CustomLidarrProxy.cs │ │ │ │ │ └── ICustomLidarrProxy.cs │ │ │ │ ├── Deezer/ │ │ │ │ │ ├── DeezerAPIService.cs │ │ │ │ │ ├── DeezerAuthService.cs │ │ │ │ │ ├── DeezerMappingHelper.cs │ │ │ │ │ ├── DeezerMetadataProxy.cs │ │ │ │ │ ├── DeezerMetadataProxySettings.cs │ │ │ │ │ ├── DeezerProxy.cs │ │ │ │ │ ├── DeezerRecords.cs │ │ │ │ │ └── IDeezerProxy.cs │ │ │ │ ├── Discogs/ │ │ │ │ │ ├── DiscogsAPIService.cs │ │ │ │ │ ├── DiscogsMappingHelper.cs │ │ │ │ │ ├── DiscogsMetadataProxy.cs │ │ │ │ │ ├── DiscogsMetadataProxySettings.cs │ │ │ │ │ ├── DiscogsProxy.cs │ │ │ │ │ ├── DiscogsRecords.cs │ │ │ │ │ └── IDiscogsProxy.cs │ │ │ │ ├── Lastfm/ │ │ │ │ │ ├── ILastfmProxy.cs │ │ │ │ │ ├── LastfmApiService.cs │ │ │ │ │ ├── LastfmImageScraper.cs │ │ │ │ │ ├── LastfmMappingHelper.cs │ │ │ │ │ ├── LastfmMetadataProxy.cs │ │ │ │ │ ├── LastfmMetadataProxySettings.cs │ │ │ │ │ ├── LastfmProxy.cs │ │ │ │ │ ├── LastfmRecordConverter.cs │ │ │ │ │ └── LastfmRecords.cs │ │ │ │ ├── MetadataProviderWrapper.cs │ │ │ │ ├── Mixed/ │ │ │ │ │ ├── AdaptiveThresholdConfig.cs │ │ │ │ │ ├── AdaptiveThresholdManager.cs │ │ │ │ │ ├── ISupportMetadataMixing.cs │ │ │ │ │ ├── MixedMetadataProxy.cs │ │ │ │ │ ├── MixedMetadataProxySettings.cs │ │ │ │ │ ├── ProxyDecisionHandler.cs │ │ │ │ │ └── ProxyMetrics.cs │ │ │ │ └── SkyHook/ │ │ │ │ ├── SkyHookMetdadataProxy.cs │ │ │ │ └── SykHookMetadataProxySettings.cs │ │ │ ├── MixedProxyBase.cs │ │ │ ├── ProxyAttribute.cs │ │ │ ├── ProxyBase.cs │ │ │ ├── ProxyService.cs │ │ │ ├── ProxyServiceStarter.cs │ │ │ ├── ProxyWrapperBase.cs │ │ │ └── RecommendArtists/ │ │ │ ├── LastFmSimilarArtistsService.cs │ │ │ ├── SimilarArtistsProxy.cs │ │ │ └── SimilarArtistsProxySettings.cs │ │ └── ScheduledTasks/ │ │ ├── IProvideScheduledTask.cs │ │ ├── ScheduledTaskBase.cs │ │ ├── ScheduledTaskService.cs │ │ ├── ScheduledTaskServiceStarter.cs │ │ └── SearchSniper/ │ │ ├── SearchSniperRepositoryHelper.cs │ │ ├── SearchSniperTask.cs │ │ └── SearchSniperTaskSettings.cs │ ├── Notifications/ │ │ ├── FlareSolverr/ │ │ │ ├── FlareDetector.cs │ │ │ ├── FlareRecords.cs │ │ │ ├── FlareSolverrHttpInterceptor.cs │ │ │ ├── FlareSolverrNotification.cs │ │ │ ├── FlareSolverrService.cs │ │ │ ├── FlareSolverrSettings.cs │ │ │ └── IFlareSolverrService.cs │ │ ├── PlaylistExport/ │ │ │ ├── PlaylistExportNotification.cs │ │ │ ├── PlaylistExportService.cs │ │ │ └── PlaylistExportSettings.cs │ │ ├── QueueCleaner/ │ │ │ ├── ImportFailureNotificationService.cs │ │ │ ├── QueueCleaner.cs │ │ │ └── QueueCleanerSettings.cs │ │ └── YouTubeProxy/ │ │ ├── YouTubeProxyNotification.cs │ │ ├── YouTubeProxyService.cs │ │ └── YouTubeProxySettings.cs │ ├── Plugin.cs │ ├── PluginInfo.targets │ ├── PluginKeys.targets │ ├── PreBuild.targets │ └── Tubifarry.csproj ├── Tubifarry.sln └── stylecop.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ ############################################################################### # Set default behavior to automatically normalize line endings. ############################################################################### * text=auto ############################################################################### # Set default behavior for command prompt diff. # # This is need for earlier builds of msysgit that does not have it on by # default for csharp files. # Note: This is only used by command line ############################################################################### #*.cs diff=csharp ############################################################################### # Set the merge driver for project and solution files # # Merging from the command prompt will add diff markers to the files if there # are conflicts (Merging from VS is not affected by the settings below, in VS # the diff markers are never inserted). Diff markers may cause the following # file extensions to fail to load in VS. An alternative would be to treat # these files as binary and thus will always conflict and require user # intervention with every merge. To do so, just uncomment the entries below ############################################################################### #*.sln merge=binary #*.csproj merge=binary #*.vbproj merge=binary #*.vcxproj merge=binary #*.vcproj merge=binary #*.dbproj merge=binary #*.fsproj merge=binary #*.lsproj merge=binary #*.wixproj merge=binary #*.modelproj merge=binary #*.sqlproj merge=binary #*.wwaproj merge=binary ############################################################################### # behavior for image files # # image files are treated as binary by default. ############################################################################### #*.jpg binary #*.png binary #*.gif binary ############################################################################### # diff behavior for common document formats # # Convert binary document formats to text before diffing them. This feature # is only available from the command line. Turn it on by uncommenting the # entries below. ############################################################################### #*.doc diff=astextplain #*.DOC diff=astextplain #*.docx diff=astextplain #*.DOCX diff=astextplain #*.dot diff=astextplain #*.DOT diff=astextplain #*.pdf diff=astextplain #*.PDF diff=astextplain #*.rtf diff=astextplain #*.RTF diff=astextplain ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: "Bug Report" description: Report a bug or unexpected behavior in Tubifarry title: "[BUG] " labels: ["bug"] assignees: [] body: - type: markdown attributes: value: | # Thank you for taking the time to report an issue! Your feedback helps us make Tubifarry better for everyone. - type: input id: tubifarry-version attributes: label: Tubifarry Version description: | What version of Tubifarry are you running? Find this in: System → Plugin. Format must be: v1.8.1 (not "latest" or "newest") placeholder: v1.8.1 validations: required: true - type: input id: lidarr-version attributes: label: Lidarr Version description: | What version of Lidarr are you using? Find this in: System → Status Format must be: v2.14.2 (not "latest") placeholder: v2.14.2 validations: required: true - type: dropdown id: os attributes: label: Operating System options: - Windows - macOS - Linux - Docker - Other (specify in additional context) validations: required: true - type: textarea id: bug-description attributes: label: Bug Description description: A clear and concise description of what the bug is and the specific feature that's not working. placeholder: | Example: "I was trying to... and then... The search function doesn't work as expected because..." Describe what you were trying to do and what went wrong: validations: required: true - type: textarea id: reproduction attributes: label: Steps to Reproduce description: Provide exact steps so we can recreate the issue placeholder: | 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error validations: required: true - type: textarea id: expected attributes: label: Expected Behavior description: What did you expect to happen? placeholder: I expected to see... validations: required: true - type: markdown attributes: value: | ## How to Enable Trace Logging 1. Go to Lidarr → Settings → General → Logging 2. Set Log Level to **Trace** (not Debug, not Info) 3. Click Save 4. Reproduce your exact problem 5. Go to System → Logs → Files tab 6. Download the most recent log file 7. Paste the relevant section below If your logs do not contain the word **`trace`**, your issue will be closed - type: textarea id: logs attributes: label: Trace Log Files description: | Paste only the relevant trace logs here. Do not delete the details tags. value: |
Logs ```plaintext Keep the plaintext markers intact and paste your logs in here... ```
validations: required: true - type: checkboxes id: log-checklist attributes: label: Log Checklist options: - label: I have enabled Trace logging in Lidarr before reproducing the issue required: true - type: textarea id: screenshots attributes: label: Screenshots description: If applicable, add screenshots to help explain your problem. value: |
Screenshots
validations: required: false - type: textarea id: additional-context attributes: label: Additional Context description: Any other information that might help us resolve the issue validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ # .github/ISSUE_TEMPLATE/config.yml blank_issues_enabled: false contact_links: - name: "Questions & Support" url: https://github.com/TypNull/Tubifarry/discussions about: Ask questions or get help with Tubifarry in our Discussions forum. - name: "Documentation" url: https://github.com/TypNull/Tubifarry/wiki about: Check out the official documentation for Tubifarry. ================================================ FILE: .github/ISSUE_TEMPLATE/documentation_improvement.yml ================================================ name: "Documentation Improvement" description: Report issues or suggest improvements for Tubifarry documentation title: "[DOCS] " labels: ["documentation"] assignees: [] body: - type: markdown attributes: value: | ## Documentation Improvement Request Thank you for helping us improve our documentation! - type: dropdown id: doc-type attributes: label: Documentation Type description: What type of documentation needs improvement? options: - Installation Guide - User Manual - API Reference - FAQ - Tutorial - Wiki Page - README - Other (specify in description) validations: required: true - type: input id: doc-location attributes: label: Documentation Location description: URL or path to the documentation that needs improvement placeholder: https://github.com/TypNull/Tubifarry/wiki/Installation or /docs/installation.md validations: required: true - type: textarea id: current-state attributes: label: Current State description: Describe what is currently in the documentation and why it needs improvement placeholder: The current documentation is missing information about... validations: required: true - type: textarea id: suggested-changes attributes: label: Suggested Changes description: Describe your suggested changes or additions placeholder: | I suggest adding a section about... The following information should be included... validations: required: true - type: textarea id: proposed-text attributes: label: Proposed Text description: If you have specific wording in mind, please provide it here value: |
Click to expand proposed text ## New Section Title Here is my suggested text for this section...
validations: required: false - type: textarea id: screenshots attributes: label: Screenshots or Examples description: If applicable, add screenshots or examples to illustrate your point value: |
Click to expand screenshots Upload or paste your screenshots here...
validations: required: false - type: checkboxes id: contribution attributes: label: Contribution options: - label: I am willing to submit a pull request with these changes required: false ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ name: "Feature Request" description: Suggest an idea or improvement for Tubifarry title: "[FEATURE] " labels: ["enhancement"] assignees: [] body: - type: markdown attributes: value: | ## Thanks for taking the time to suggest a feature! Please provide detailed information about your feature idea. Check existing issues first to avoid duplicates. - type: textarea id: feature-request attributes: label: Feature Request description: Describe what you'd like to see added and why it would be valuable. placeholder: | I would like Tubifarry to support [specific feature] because [reason]. This would improve the experience by... The feature should work like [detailed description]. validations: required: true - type: dropdown id: importance attributes: label: Priority Level description: How important is this feature for your workflow? options: - Nice to have (low priority) - Would improve my experience (medium priority) - Critical to my use case (high priority) validations: required: true - type: textarea id: mockups attributes: label: Visual Designs or Mockups description: If you have any mockups, wireframes, or designs, please share them here. value: |
Mockups
validations: required: false - type: dropdown id: contribution attributes: label: Are you willing to contribute to this feature? description: Let us know if you'd be interested in implementing this feature options: - No, I'm just suggesting the idea - I might contribute, but I'd need guidance - Yes, I'd be willing to implement this feature validations: required: true - type: textarea id: technical attributes: label: Technical Details (Optional) description: If you have technical expertise or ideas for implementation, feel free to share. value: |
Technical Details
validations: required: false - type: checkboxes id: prerequisites attributes: label: Prerequisites options: - label: I have searched existing issues to ensure this is not a duplicate required: true ================================================ FILE: .github/workflows/branch-detection.yml ================================================ name: Detect Branch from Tag or Reference on: workflow_call: inputs: github_ref: required: true type: string description: "The GitHub reference (usually github.ref)" outputs: branch_name: description: "The detected branch name" value: ${{ jobs.detect.outputs.branch_name }} jobs: detect: runs-on: ubuntu-latest outputs: branch_name: ${{ steps.detect_branch.outputs.branch_name }} steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - name: Ensure all Git refs are available run: | git fetch --prune --unshallow || git fetch --prune git fetch --tags --force echo "Available remote branches:" git branch -r echo "Available tags:" git tag -l - name: Detect Branch id: detect_branch run: | # Enhanced branch detection from tag names if [[ "${{ inputs.github_ref }}" == refs/tags/* ]]; then TAG_NAME=${{ inputs.github_ref }} TAG_NAME=${TAG_NAME#refs/tags/} DEFAULT_BRANCH=$(git remote show origin | grep 'HEAD branch' | cut -d' ' -f5) # Check if tag contains branch info in format v1.2.3-branch-name if [[ "$TAG_NAME" == v*-* ]]; then # Extract potential branch name after the dash TAG_BRANCH=${TAG_NAME#*-} echo "Detected potential branch suffix in tag: $TAG_BRANCH" # List all available branches and look for best match AVAILABLE_BRANCHES=$(git branch -r | grep -v HEAD | sed -e 's/ *origin\///g') echo "Available branches:" echo "$AVAILABLE_BRANCHES" # First, try exact match if echo "$AVAILABLE_BRANCHES" | grep -q "^$TAG_BRANCH$"; then BRANCH_NAME="$TAG_BRANCH" echo "Found exact branch match: $BRANCH_NAME" # Then try to match longer branch names that contain the tag suffix elif echo "$AVAILABLE_BRANCHES" | grep -q "$TAG_BRANCH"; then BRANCH_NAME=$(echo "$AVAILABLE_BRANCHES" | grep "$TAG_BRANCH" | head -n 1) echo "Found branch containing tag suffix: $BRANCH_NAME" fi fi # If no branch name from tag, try to find branch that contains this tag if [ -z "$BRANCH_NAME" ]; then echo "No branch found from tag suffix, trying git branch --contains..." CONTAINING_BRANCHES=$(git branch -r --contains $TAG_NAME | grep -v HEAD | sed -e 's/ *origin\///g') echo "Branches containing this tag: $CONTAINING_BRANCHES" # Prioritize master/main when multiple branches contain the commit if echo "$CONTAINING_BRANCHES" | grep -qE "^(master|main)$"; then BRANCH_NAME=$(echo "$CONTAINING_BRANCHES" | grep -E "^(master|main)$" | head -n 1) echo "Prioritizing default branch: $BRANCH_NAME" elif echo "$CONTAINING_BRANCHES" | grep -q "^$DEFAULT_BRANCH$"; then BRANCH_NAME="$DEFAULT_BRANCH" echo "Using repository default branch: $BRANCH_NAME" else BRANCH_NAME=$(echo "$CONTAINING_BRANCHES" | head -n 1) echo "Using first available branch: $BRANCH_NAME" fi fi # Fallback to the default branch if no branch is found if [ -z "$BRANCH_NAME" ]; then BRANCH_NAME=$DEFAULT_BRANCH echo "Tag not associated with branch, using default branch: $BRANCH_NAME" fi else # When triggered by a push, use GITHUB_REF directly BRANCH_NAME=${{ inputs.github_ref }} BRANCH_NAME=${BRANCH_NAME#refs/heads/} fi echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT echo "Detected branch: $BRANCH_NAME" ================================================ FILE: .github/workflows/compilation.yml ================================================ name: Compile Project on: workflow_call: inputs: dotnet_version: required: true type: string description: ".NET SDK version to use" plugin_name: required: true type: string description: "Name of the plugin" package_version: required: true type: string description: "Version number for the package" framework: required: true type: string description: "Framework version extracted from project file" git_commit: required: true type: string description: "Git commit SHA used for the build" git_branch: required: true type: string description: "Git branch name" git_tag: required: false type: string description: "Git tag if available" default: "" repo_url: required: true type: string description: "Base repository URL" branch_repo_url: required: true type: string description: "Repository URL with branch path" repo_owner: required: true type: string description: "Repository owner" minimum_lidarr_version: required: false type: string description: "Minimum Lidarr version required" default: "" secrets: SPOTIFY_CLIENT_ID: required: false description: "Spotify API Client ID" SPOTIFY_CLIENT_SECRET: required: false description: "Spotify API Client Secret" YOUTUBE_SECRET: required: false description: "YouTube API Secret" SENTRY_DSN: required: false description: "Sentry DSN for error reporting" outputs: build_status: description: "Status of the build" value: ${{ jobs.compile.outputs.build_status }} plugin_output_path: description: "Path to the built plugin output directory" value: ${{ jobs.compile.outputs.plugin_output_path }} jobs: compile: runs-on: ubuntu-latest outputs: build_status: ${{ steps.build_result.outputs.status }} plugin_output_path: ${{ steps.find_output.outputs.plugin_path }} steps: - name: Checkout with submodules uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 - name: Setup .NET Core uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ inputs.dotnet_version }} # Add NuGet package caching - name: Cache NuGet packages uses: actions/cache@v3 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} restore-keys: | ${{ runner.os }}-nuget- - name: Create global.json run: | echo '{"sdk":{"version": "${{ inputs.dotnet_version }}"}}' > ./global.json - name: Create Release Notes id: release_notes run: | mkdir -p ./Properties cat > ./Properties/release_notes.txt << EOL Version: ${{ inputs.package_version }} Branch: ${{ inputs.git_branch }} Commit: ${{ inputs.git_commit }} Framework: ${{ inputs.framework }} Repository: ${{ inputs.branch_repo_url }} Minimum Lidarr Version: ${{ inputs.minimum_lidarr_version }} Built at: $(date -u +"%Y-%m-%d %H:%M:%S UTC") Plugin Info: Name: ${{ inputs.plugin_name }} Owner: ${{ inputs.repo_owner }} EOL echo "release_notes=./Properties/release_notes.txt" >> $GITHUB_OUTPUT - name: Display Build Parameters run: | echo "Plugin Name: ${{ inputs.plugin_name }}" echo "Package Version: ${{ inputs.package_version }}" echo "Git Branch: ${{ inputs.git_branch }}" echo "Git Commit: ${{ inputs.git_commit }}" echo "Git Tag: ${{ inputs.git_tag }}" echo "Repo URL: ${{ inputs.repo_url }}" echo "Branch Repo URL: ${{ inputs.branch_repo_url }}" echo "Repo Owner: ${{ inputs.repo_owner }}" echo "Release Notes Path: ${{ steps.release_notes.outputs.release_notes }}" echo "Minimum Lidarr Version: ${{ inputs.minimum_lidarr_version }}" echo "Spotify Client ID is configured: ${{ secrets.SPOTIFY_CLIENT_ID != '' }}" echo "Spotify Client Secret is configured: ${{ secrets.SPOTIFY_CLIENT_SECRET != '' }}" echo "YouTube Secret is configured: ${{ secrets.YOUTUBE_SECRET != '' }}" echo "Sentry DSN is configured: ${{ secrets.SENTRY_DSN != '' }}" - name: Build with package version and metadata id: build_step env: SPOTIFY_CLIENT_ID: ${{ secrets.SPOTIFY_CLIENT_ID }} SPOTIFY_CLIENT_SECRET: ${{ secrets.SPOTIFY_CLIENT_SECRET }} YOUTUBE_SECRET: ${{ secrets.YOUTUBE_SECRET }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} run: | # Restore NuGet packages first dotnet restore *.sln # Then build with all parameters dotnet build *.sln -c Release -f ${{ inputs.framework }} \ -p:Version=${{ inputs.package_version }} \ -p:AssemblyVersion=${{ inputs.package_version }} \ -p:FileVersion=${{ inputs.package_version }} \ -p:Branch="${{ inputs.git_branch }}" \ -p:GitCommit="${{ inputs.git_commit }}" \ -p:GitTag="${{ inputs.git_tag }}" \ -p:RepoUrl="${{ inputs.branch_repo_url }}" \ -p:Author="${{ inputs.repo_owner }}" \ -p:ReleaseNotesFile="${{ steps.release_notes.outputs.release_notes }}" \ -p:MinimumLidarrVersion="${{ inputs.minimum_lidarr_version }}" \ -p:SpotifyClientId=$SPOTIFY_CLIENT_ID \ -p:SpotifyClientSecret=$SPOTIFY_CLIENT_SECRET \ -p:YouTubeSecret=$YOUTUBE_SECRET \ -p:SentryDsn=$SENTRY_DSN \ -p:CI="true" \ -v:n - name: Find Plugin Output Directory id: find_output run: | # More thorough search for the plugin output directory echo "Searching for plugin output directories..." find . -name "_plugins" -type d # Find the plugin output directory PLUGIN_OUTPUT_DIR=$(find . -type d -path "*/_plugins/*/${{ inputs.plugin_name }}" | head -n 1) if [ -z "$PLUGIN_OUTPUT_DIR" ]; then echo "Trying alternate search pattern..." PLUGIN_OUTPUT_DIR=$(find . -type d -path "*/_plugins/*" -name "${{ inputs.plugin_name }}" | head -n 1) fi # If still not found, use the first plugin directory we can find if [ -z "$PLUGIN_OUTPUT_DIR" ]; then echo "Plugin directory not found by name. Using first available plugin directory..." FIRST_PLUGINS_DIR=$(find . -type d -path "*/_plugins/*" | sort | head -n 1) if [ -n "$FIRST_PLUGINS_DIR" ]; then PLUGIN_OUTPUT_DIR=$FIRST_PLUGINS_DIR echo "Using plugin directory: $PLUGIN_OUTPUT_DIR" else echo "::error::No plugin directories found at all! Build may have failed to produce plugin files." exit 1 fi fi echo "Plugin output directory: $PLUGIN_OUTPUT_DIR" # Check if directory contains any files at all FILES_COUNT=$(find "$PLUGIN_OUTPUT_DIR" -type f | wc -l) if [ "$FILES_COUNT" -eq 0 ]; then echo "::error::Directory is empty! Build may have failed to produce plugin files." exit 1 fi echo "Found $FILES_COUNT files in $PLUGIN_OUTPUT_DIR" echo "plugin_path=$PLUGIN_OUTPUT_DIR" >> $GITHUB_OUTPUT # Copy release notes to output directory cp ${{ steps.release_notes.outputs.release_notes }} $PLUGIN_OUTPUT_DIR/release_notes.txt - name: Record Build Result id: build_result run: | echo "status=success" >> $GITHUB_OUTPUT # Upload the actual plugin files - name: Upload Plugin Artifact uses: actions/upload-artifact@v4 with: name: plugin-output path: ${{ steps.find_output.outputs.plugin_path }} retention-days: 1 - name: Upload Build Log if: always() uses: actions/upload-artifact@v4 with: name: build-logs path: | ./**/*.binlog ./**/*.log retention-days: 1 if-no-files-found: warn ================================================ FILE: .github/workflows/dotnet-setup.yml ================================================ name: Setup .NET Environment on: workflow_call: inputs: dotnet_version: required: true type: string description: ".NET SDK version to use" framework: required: false type: string description: "Framework version" default: "net6.0" outputs: submodules_ready: description: "Whether submodules were initialized successfully" value: ${{ jobs.setup.outputs.submodules_ready }} jobs: setup: runs-on: ubuntu-latest outputs: submodules_ready: ${{ steps.check_submodules.outputs.ready }} steps: - name: Checkout with submodules uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 - name: Setup .NET Core uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ inputs.dotnet_version }} # Add NuGet package caching - name: Cache NuGet packages uses: actions/cache@v3 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} restore-keys: | ${{ runner.os }}-nuget- - name: Create global.json run: | echo '{"sdk":{"version": "${{ inputs.dotnet_version }}"}}' > ./global.json - name: Initialize and verify submodules id: check_submodules run: | echo "Listing Submodules directory:" ls -la Submodules/ || echo "Submodules directory not found" if [ -d "Submodules/Lidarr" ]; then echo "Listing Lidarr submodule:" ls -la Submodules/Lidarr/ echo "Listing Lidarr source directory:" ls -la Submodules/Lidarr/src/ || echo "Lidarr src directory not found" else echo "Lidarr submodule not found" fi echo "Checking .gitmodules file:" cat .gitmodules || echo ".gitmodules file not found" # Initialize submodules if they weren't checked out properly if [ ! -d "Submodules/Lidarr/src" ]; then echo "Manually initializing submodules..." git submodule update --init --recursive fi # Verify that submodules are now properly initialized if [ -d "Submodules/Lidarr/src" ]; then echo "ready=true" >> $GITHUB_OUTPUT echo "Submodules successfully initialized" else echo "ready=false" >> $GITHUB_OUTPUT echo "Warning: Submodules could not be initialized" fi ================================================ FILE: .github/workflows/git-info.yml ================================================ name: Extract Git Information on: workflow_call: inputs: tag_ref: required: true type: string description: "The tag reference (usually github.ref)" branch_name: required: true type: string description: "Branch name (passed from main workflow)" outputs: git_commit: description: "Git commit SHA used for the build" value: ${{ jobs.git_info.outputs.git_commit }} git_branch: description: "Git branch name" value: ${{ jobs.git_info.outputs.git_branch }} git_tag: description: "Git tag if available" value: ${{ jobs.git_info.outputs.git_tag }} repo_url: description: "Repository URL (base URL without branch)" value: ${{ jobs.git_info.outputs.repo_url }} branch_repo_url: description: "Repository URL with branch path for non-master branches" value: ${{ jobs.git_info.outputs.branch_repo_url }} repo_owner: description: "Repository owner" value: ${{ jobs.git_info.outputs.repo_owner }} commit_messages: description: "Commit messages since last release" value: ${{ jobs.git_info.outputs.commit_messages }} is_latestrelease: description: "Whether this should be the newest stable release" value: ${{ jobs.git_info.outputs.is_latestrelease }} plugin_input_format: description: "Plugin input format: name@owner#branch" value: ${{ jobs.git_info.outputs.plugin_input_format }} jobs: git_info: runs-on: ubuntu-latest outputs: git_commit: ${{ steps.extract_git_info.outputs.git_commit }} git_branch: ${{ steps.extract_git_info.outputs.git_branch }} git_tag: ${{ steps.extract_git_info.outputs.git_tag }} repo_url: ${{ steps.extract_git_info.outputs.repo_url }} branch_repo_url: ${{ steps.extract_git_info.outputs.branch_repo_url }} repo_owner: ${{ steps.extract_git_info.outputs.repo_owner }} commit_messages: ${{ steps.get_commit_messages.outputs.commit_messages }} is_latestrelease: ${{ steps.check_latest_release.outputs.is_latestrelease }} plugin_input_format: ${{ steps.extract_git_info.outputs.plugin_input_format }} steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - name: Fetch tags run: | git fetch --tags --force - name: Extract Git Information id: extract_git_info run: | # Extract commit hash GIT_COMMIT=$(git rev-parse --short HEAD) echo "git_commit=$GIT_COMMIT" >> $GITHUB_OUTPUT echo "Git commit: $GIT_COMMIT" # Use branch name passed from main workflow BRANCH_NAME="${{ inputs.branch_name }}" echo "git_branch=$BRANCH_NAME" >> $GITHUB_OUTPUT echo "Using branch name: $BRANCH_NAME" # Extract tag name GIT_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") echo "git_tag=$GIT_TAG" >> $GITHUB_OUTPUT echo "Git tag: $GIT_TAG" # Extract repository URL REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" echo "repo_url=$REPO_URL" >> $GITHUB_OUTPUT echo "Repository URL: $REPO_URL" # Create branch-specific repository URL (master/main uses base URL) if [[ "$BRANCH_NAME" != "master" && "$BRANCH_NAME" != "main" ]]; then BRANCH_REPO_URL="$REPO_URL/tree/$BRANCH_NAME" echo "Branch-specific URL: $BRANCH_REPO_URL" else BRANCH_REPO_URL="$REPO_URL" echo "Using base URL for master/main branch: $BRANCH_REPO_URL" fi echo "branch_repo_url=$BRANCH_REPO_URL" >> $GITHUB_OUTPUT # Extract repository owner REPO_OWNER=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f1) echo "repo_owner=$REPO_OWNER" >> $GITHUB_OUTPUT echo "Repository owner: $REPO_OWNER" # Generate plugin input format: name@owner#branch (or name@owner if master/main) REPO_NAME=$(basename $GITHUB_REPOSITORY) if [[ "$BRANCH_NAME" != "master" && "$BRANCH_NAME" != "main" ]]; then PLUGIN_INPUT="$REPO_NAME@$REPO_OWNER#$BRANCH_NAME" else PLUGIN_INPUT="$REPO_NAME@$REPO_OWNER" fi echo "plugin_input_format=$PLUGIN_INPUT" >> $GITHUB_OUTPUT echo "Plugin input format: $PLUGIN_INPUT" - name: Check if Latest Release id: check_latest_release run: | BRANCH_NAME="${{ inputs.branch_name }}" CURRENT_TAG="${{ inputs.tag_ref }}" CURRENT_TAG="${CURRENT_TAG#refs/tags/}" VERSION_NUMBER="${CURRENT_TAG#v}" # Only master/main with non-0.x versions are latest releases if [[ ("$BRANCH_NAME" == "master" || "$BRANCH_NAME" == "main") && ! "$VERSION_NUMBER" =~ ^0\. ]]; then echo "is_latestrelease=true" >> $GITHUB_OUTPUT else echo "is_latestrelease=false" >> $GITHUB_OUTPUT fi echo "Branch: $BRANCH_NAME" echo "Version: $VERSION_NUMBER" echo "Is latest release: $(cat $GITHUB_OUTPUT | grep is_latestrelease | cut -d= -f2)" - name: Get Commit Messages id: get_commit_messages run: | # Get the current tag from inputs.tag_ref CURRENT_TAG="${{ inputs.tag_ref }}" CURRENT_TAG="${CURRENT_TAG#refs/tags/}" echo "Current tag: $CURRENT_TAG" # Get a list of all tags sorted by creation date (newest first) ALL_TAGS=$(git tag --sort=-creatordate) echo "All tags:" echo "$ALL_TAGS" # Find the previous tag (the tag after the current one in the sorted list) PREVIOUS_TAG="" FOUND_CURRENT=false for tag in $ALL_TAGS; do if $FOUND_CURRENT; then PREVIOUS_TAG=$tag break fi if [ "$tag" = "$CURRENT_TAG" ]; then FOUND_CURRENT=true fi done echo "Previous tag found: $PREVIOUS_TAG" # Get commit messages between tags or all commits if no previous tag if [ -z "$PREVIOUS_TAG" ]; then echo "No previous tag found. Retrieving all commit messages." COMMIT_MESSAGES=$(git log --pretty=format:"- %h %s" -n 20) else echo "Getting commits between $PREVIOUS_TAG and $CURRENT_TAG" COMMIT_MESSAGES=$(git log --pretty=format:"- %h %s" $PREVIOUS_TAG..$CURRENT_TAG) fi # Check if we got any commit messages if [ -z "$COMMIT_MESSAGES" ]; then echo "No commit messages found!" COMMIT_MESSAGES="- No significant changes detected between releases." fi # Debug: Show the commit messages echo "Commit messages:" echo "$COMMIT_MESSAGES" # No need to base64 encode - we'll pass it directly echo "commit_messages<> $GITHUB_OUTPUT echo "$COMMIT_MESSAGES" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT ================================================ FILE: .github/workflows/main.yml ================================================ name: Build Plugin on: push: tags: - 'v*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: write env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true DOTNET_CLI_TELEMETRY_OPTOUT: true jobs: # Detect branch from tag or reference branch-detection: uses: ./.github/workflows/branch-detection.yml with: github_ref: ${{ github.ref }} # Extract metadata from the project metadata: uses: ./.github/workflows/metadata.yml needs: [branch-detection] with: override_plugin_name: 'Tubifarry' branch_name: ${{ needs.branch-detection.outputs.branch_name }} # Extract git-specific information git-info: uses: ./.github/workflows/git-info.yml needs: [branch-detection] with: tag_ref: ${{ github.ref }} branch_name: ${{ needs.branch-detection.outputs.branch_name }} # Setup .NET environment and submodules dotnet-setup: uses: ./.github/workflows/dotnet-setup.yml with: dotnet_version: '8.0.404' framework: ${{ needs.metadata.outputs.framework }} needs: [metadata] # Compile the project compilation: uses: ./.github/workflows/compilation.yml with: dotnet_version: '8.0.404' plugin_name: ${{ needs.metadata.outputs.plugin_name }} package_version: ${{ needs.metadata.outputs.package_version }} framework: ${{ needs.metadata.outputs.framework }} git_commit: ${{ needs.git-info.outputs.git_commit }} git_branch: ${{ needs.git-info.outputs.git_branch }} git_tag: ${{ needs.git-info.outputs.git_tag }} repo_url: ${{ needs.git-info.outputs.repo_url }} branch_repo_url: ${{ needs.git-info.outputs.branch_repo_url }} repo_owner: ${{ needs.git-info.outputs.repo_owner }} minimum_lidarr_version: ${{ needs.metadata.outputs.minimum_lidarr_version }} secrets: SPOTIFY_CLIENT_ID: ${{ secrets.SPOTIFY_CLIENT_ID }} SPOTIFY_CLIENT_SECRET: ${{ secrets.SPOTIFY_CLIENT_SECRET }} YOUTUBE_SECRET: ${{ secrets.YOUTUBE_SECRET }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} needs: [metadata, git-info, dotnet-setup] # Package the artifacts packaging: uses: ./.github/workflows/packaging.yml with: plugin_name: ${{ needs.metadata.outputs.plugin_name }} package_version: ${{ needs.metadata.outputs.package_version }} build_suffix: ${{ needs.metadata.outputs.build_suffix }} build_status: ${{ needs.compilation.outputs.build_status }} needs: [metadata, compilation] # Generate release notes release-notes: uses: ./.github/workflows/release-notes.yml with: plugin_name: ${{ needs.metadata.outputs.plugin_name }} package_version: ${{ needs.metadata.outputs.package_version }} build_suffix: ${{ needs.metadata.outputs.build_suffix }} minimum_lidarr_version: ${{ needs.metadata.outputs.minimum_lidarr_version }} commit_messages: ${{ needs.git-info.outputs.commit_messages }} git_commit: ${{ needs.git-info.outputs.git_commit }} branch_repo_url: ${{ needs.git-info.outputs.branch_repo_url }} git_branch: ${{ needs.git-info.outputs.git_branch }} plugin_input_format: ${{ needs.git-info.outputs.plugin_input_format }} needs: [metadata, git-info] # Publish the release publishing: uses: ./.github/workflows/publishing.yml with: plugin_name: ${{ needs.metadata.outputs.plugin_name }} package_version: ${{ needs.metadata.outputs.package_version }} build_suffix: ${{ needs.metadata.outputs.build_suffix }} is_prerelease: ${{ needs.metadata.outputs.is_prerelease }} is_latestrelease: ${{ needs.git-info.outputs.is_latestrelease }} release_notes_id: ${{ needs.release-notes.outputs.release_notes_id }} git_commit: ${{ needs.git-info.outputs.git_commit }} branch_repo_url: ${{ needs.git-info.outputs.branch_repo_url }} git_branch: ${{ needs.git-info.outputs.git_branch }} needs: [metadata, git-info, compilation, packaging, release-notes] if: needs.compilation.outputs.build_status == 'success' ================================================ FILE: .github/workflows/metadata.yml ================================================ name: Extract Project Metadata on: workflow_call: inputs: override_plugin_name: required: false type: string description: "Override the plugin name if different from repository name" default: "" branch_name: required: true type: string description: "Branch name (passed from main workflow)" outputs: plugin_name: description: "The name of the plugin" value: ${{ jobs.metadata.outputs.plugin_name }} package_version: description: "The version of the package" value: ${{ jobs.metadata.outputs.package_version }} build_suffix: description: "Build suffix based on framework and branch name" value: ${{ jobs.metadata.outputs.build_suffix }} minimum_lidarr_version: description: "Minimum Lidarr version required" value: ${{ jobs.metadata.outputs.minimum_lidarr_version }} is_prerelease: description: "Whether this is a prerelease" value: ${{ jobs.metadata.outputs.is_prerelease }} framework: description: "Framework version extracted from project file" value: ${{ jobs.metadata.outputs.framework }} jobs: metadata: runs-on: ubuntu-latest outputs: plugin_name: ${{ steps.extract_repo_name.outputs.plugin_name }} package_version: ${{ steps.extract_version.outputs.package_version }} build_suffix: ${{ steps.build_info.outputs.build_suffix }} branch_name: ${{ inputs.branch_name }} framework: ${{ steps.extract_framework.outputs.framework }} minimum_lidarr_version: ${{ steps.fetch_minimum_lidarr_version.outputs.minimum_lidarr_version }} is_prerelease: ${{ steps.extract_version.outputs.is_prerelease }} steps: - name: Checkout uses: actions/checkout@v4 with: submodules: true fetch-depth: 0 - name: Extract repository name or use override id: extract_repo_name run: | # Check if override_plugin_name is provided if [ -n "${{ inputs.override_plugin_name }}" ]; then PLUGIN_NAME="${{ inputs.override_plugin_name }}" echo "Using provided plugin name override: $PLUGIN_NAME" else # Fall back to repository name PLUGIN_NAME=$(basename $GITHUB_REPOSITORY) echo "Using repository name as plugin name: $PLUGIN_NAME" fi echo "plugin_name=$PLUGIN_NAME" >> $GITHUB_OUTPUT - name: Extract framework from project file id: extract_framework run: | # Get plugin name PLUGIN_NAME="${{ steps.extract_repo_name.outputs.plugin_name }}" echo "Looking for project file: $PLUGIN_NAME/$PLUGIN_NAME.csproj" # Check if the main project file exists if [ -f "$PLUGIN_NAME/$PLUGIN_NAME.csproj" ]; then PROJECT_FILE="$PLUGIN_NAME/$PLUGIN_NAME.csproj" else echo "Main project file not found, searching for any .csproj file..." # Find the first .csproj file in the repo PROJECT_FILE=$(find . -name "*.csproj" | head -n 1) fi if [ -n "$PROJECT_FILE" ]; then echo "Found project file: $PROJECT_FILE" # Extract the TargetFramework using grep and sed FRAMEWORK=$(grep -o '.*' "$PROJECT_FILE" | sed 's/\(.*\)<\/TargetFramework>/\1/') echo "Extracted framework: $FRAMEWORK" else echo "No .csproj files found, defaulting to net6.0" FRAMEWORK="net6.0" fi # If framework is still empty, default to net6.0 if [ -z "$FRAMEWORK" ]; then echo "Framework not found in project file, defaulting to net6.0" FRAMEWORK="net6.0" fi echo "Using framework: $FRAMEWORK" echo "framework=$FRAMEWORK" >> $GITHUB_OUTPUT - name: Extract version from tag id: extract_version run: | TAG_VERSION=${GITHUB_REF#refs/tags/v} # Remove branch suffix if present if [[ "$TAG_VERSION" == *-* ]]; then VERSION_ONLY=${TAG_VERSION%%-*} echo "Extracted version without branch suffix: $VERSION_ONLY" TAG_VERSION=$VERSION_ONLY fi echo "Extracted version: $TAG_VERSION" echo "package_version=$TAG_VERSION" >> $GITHUB_OUTPUT if [[ "$TAG_VERSION" == 0.* ]]; then echo "is_prerelease=true" >> $GITHUB_OUTPUT else echo "is_prerelease=false" >> $GITHUB_OUTPUT fi - name: Determine build suffix id: build_info run: | # Get the framework from the extract_framework step FRAMEWORK="${{ steps.extract_framework.outputs.framework }}" # Build suffix is just the framework BUILD_SUFFIX="$FRAMEWORK" echo "Framework: $FRAMEWORK" echo "Build suffix: $BUILD_SUFFIX" echo "build_suffix=$BUILD_SUFFIX" >> $GITHUB_OUTPUT - name: Fetch Minimum Lidarr Version from Submodule id: fetch_minimum_lidarr_version run: | if [ -f "Submodules/Lidarr/azure-pipelines.yml" ]; then RAW_YAML=$(cat Submodules/Lidarr/azure-pipelines.yml) else echo "Error: Submodules/Lidarr/azure-pipelines.yml not found!" exit 1 fi MAJOR_VERSION=$(echo "$RAW_YAML" | grep "majorVersion:" | head -n 1 | sed "s/.*majorVersion: *'\([^']*\)'.*/\1/") echo "Extracted majorVersion: $MAJOR_VERSION" DOT_COUNT=$(echo "$MAJOR_VERSION" | awk -F. '{print NF-1}') if [ "$DOT_COUNT" -eq 2 ]; then MINIMUM_LIDARR_VERSION="${MAJOR_VERSION}.0" else MINIMUM_LIDARR_VERSION="$MAJOR_VERSION" fi echo "Minimum Lidarr Version: $MINIMUM_LIDARR_VERSION" echo "minimum_lidarr_version=$MINIMUM_LIDARR_VERSION" >> $GITHUB_OUTPUT ================================================ FILE: .github/workflows/packaging.yml ================================================ name: Package Plugin on: workflow_call: inputs: plugin_name: required: true type: string description: "Name of the plugin" package_version: required: true type: string description: "Version number for the package" build_suffix: required: true type: string description: "Build suffix based on framework and branch" build_status: required: true type: string description: "Status of the build step" outputs: package_path: description: "Path to the final package" value: ${{ jobs.package.outputs.package_path }} jobs: package: runs-on: ubuntu-latest outputs: package_path: ${{ steps.zip_plugin.outputs.package_path }} steps: - name: Checkout repository uses: actions/checkout@v4 - name: Download built plugin uses: actions/download-artifact@v4 with: name: plugin-output path: ./plugin-output # Don't continue on error - must have plugin to package - name: Verify plugin artifacts are available run: | echo "Contents of the downloaded plugin output directory:" find ./plugin-output -type f | sort # Fail if no plugin files found PLUGIN_FILES_COUNT=$(find ./plugin-output -name "*.${{ inputs.plugin_name }}.*" -o -name "*.Plugin.${{ inputs.plugin_name }}.*" | wc -l) if [ "$PLUGIN_FILES_COUNT" -eq 0 ]; then echo "::error::No plugin files found in downloaded artifacts! Build may have failed to produce plugin files." exit 1 fi echo "Found $PLUGIN_FILES_COUNT plugin files." - name: Zip Plugin id: zip_plugin run: | cd ./plugin-output PACKAGE_NAME="${{ inputs.plugin_name }}-v${{ inputs.package_version }}.${{ inputs.build_suffix }}.zip" # List files that will be included in the package echo "Files to be packaged:" find . -name "*.${{ inputs.plugin_name }}.*" -o -name "*.Plugin.${{ inputs.plugin_name }}.*" # Create zip with all plugin files and release notes find . -name "*.${{ inputs.plugin_name }}.*" -o -name "*.Plugin.${{ inputs.plugin_name }}.*" | xargs zip -r $PACKAGE_NAME # Add release notes to the zip if it exists if [ -f "release_notes.txt" ]; then zip -r $PACKAGE_NAME release_notes.txt fi # Verify the zip was created and has content if [ ! -f "$PACKAGE_NAME" ] || [ $(stat -c%s "$PACKAGE_NAME") -eq 0 ]; then echo "::error::Failed to create package or package is empty!" exit 1 fi echo "Created package: $PACKAGE_NAME" echo "Contents of zip file:" unzip -l $PACKAGE_NAME echo "package_path=./plugin-output/$PACKAGE_NAME" >> $GITHUB_OUTPUT - name: Upload Package Artifact uses: actions/upload-artifact@v4 with: name: plugin-package path: ${{ steps.zip_plugin.outputs.package_path }} retention-days: 7 ================================================ FILE: .github/workflows/publishing.yml ================================================ name: Publish GitHub Release on: workflow_call: inputs: plugin_name: required: true type: string description: "Name of the plugin" package_version: required: true type: string description: "Version number for the release" build_suffix: required: true type: string description: "Build suffix based on framework and branch" is_prerelease: required: true type: string description: "Whether this is a pre-release" is_latestrelease: required: true type: string description: "Whether this should be the newest stable release" release_notes_id: required: true type: string description: "ID of the generated release notes" git_commit: required: true type: string description: "Git commit SHA used for the build" branch_repo_url: required: false type: string description: "Repository URL with branch path" default: "" git_branch: required: true type: string description: "Git branch name for target_commitish" outputs: release_url: description: "URL of the created release" value: ${{ jobs.publish.outputs.release_url }} jobs: publish: runs-on: ubuntu-latest outputs: release_url: ${{ steps.create_release.outputs.url }} steps: - name: Checkout repository uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v4 with: name: plugin-package path: ./artifacts - name: Download release notes uses: actions/download-artifact@v4 with: name: ${{ inputs.release_notes_id }} path: ./notes - name: Verify artifacts id: verify_artifacts run: | echo "Artifacts directory contents:" find ./artifacts -type f | sort # The artifact should follow the naming convention with framework from csproj ARTIFACT_PATH="./artifacts/${{ inputs.plugin_name }}-v${{ inputs.package_version }}.${{ inputs.build_suffix }}.zip" if [ ! -f "$ARTIFACT_PATH" ]; then echo "::error::Artifact not found at expected path: $ARTIFACT_PATH" exit 1 fi echo "artifact_path=$ARTIFACT_PATH" >> $GITHUB_OUTPUT FILESIZE=$(stat -c%s "$ARTIFACT_PATH") echo "Artifact size: $FILESIZE bytes" echo "artifact_size=$FILESIZE" >> $GITHUB_OUTPUT # Check release notes if [ ! -f "./notes/release_notes.md" ]; then echo "::error::Release notes not found at expected path: ./notes/release_notes.md" exit 1 fi echo "release_notes_path=./notes/release_notes.md" >> $GITHUB_OUTPUT - name: Create GitHub Release id: create_release uses: softprops/action-gh-release@v2 with: tag_name: ${{ github.ref }} name: "🚀 Release ${{ inputs.package_version }}" body_path: ${{ steps.verify_artifacts.outputs.release_notes_path }} prerelease: ${{ inputs.is_prerelease == 'true' }} make_latest: ${{ inputs.is_latestrelease == 'true' }} target_commitish: ${{ inputs.git_branch }} files: | ${{ steps.verify_artifacts.outputs.artifact_path }} fail_on_unmatched_files: true env: GITHUB_TOKEN: ${{ github.token }} - name: Verify release was created run: | echo "Release URL: ${{ steps.create_release.outputs.url }}" if [ -z "${{ steps.create_release.outputs.url }}" ]; then echo "::warning::Release URL is empty, there might have been an issue with release creation" else echo "Release successfully created" fi - name: Notify of successful release if: success() run: | echo "::notice::🎉 Successfully published release v${{ inputs.package_version }} for ${{ inputs.plugin_name }}" # Include branch-specific URL in notification if available if [ -n "${{ inputs.branch_repo_url }}" ]; then echo "Repository: ${{ inputs.branch_repo_url }}" fi echo "Release available at: ${{ steps.create_release.outputs.url }}" ================================================ FILE: .github/workflows/release-notes.yml ================================================ name: Generate Release Notes on: workflow_call: inputs: plugin_name: required: true type: string description: "Name of the plugin" package_version: required: true type: string description: "Version number for the release" build_suffix: required: true type: string description: "Build suffix based on framework and branch" minimum_lidarr_version: required: true type: string description: "Minimum required Lidarr version" commit_messages: required: true type: string description: "Commit messages since last release" git_commit: required: true type: string description: "Git commit SHA used for the build" branch_repo_url: required: true type: string description: "Repository URL with branch path" git_branch: required: true type: string description: "Current git branch" plugin_input_format: required: true type: string description: "Plugin input format: name@owner#branch" outputs: release_notes_id: description: "ID of the generated release notes" value: ${{ jobs.generate.outputs.release_notes_id }} jobs: generate: runs-on: ubuntu-latest outputs: release_notes_id: ${{ steps.create_notes.outputs.release_notes_id }} steps: - name: Generate Release Notes id: create_notes run: | # Generate a unique ID for this release notes NOTES_ID="release-notes-${{ github.run_id }}" echo "release_notes_id=$NOTES_ID" >> $GITHUB_OUTPUT # Print the commit messages for debugging echo "Commit messages:" echo "${{ inputs.commit_messages }}" # Use the git_branch input directly BRANCH_NAME="${{ inputs.git_branch }}" echo "Using branch from git-info: $BRANCH_NAME" # Set default branch DEFAULT_BRANCH="master" # Generate release notes in markdown cat > release_notes.md << EOL ## 📝 What's New: ${{ inputs.commit_messages }} --- ## 📥 Installation Notes: - In Lidarr, navigate to **System -> Plugins** - Enter: \`${{ inputs.plugin_input_format }}\` --- ## 📦 Package Information **Version:** ${{ inputs.package_version }} **.NET Version:** ${{ inputs.build_suffix }} **Minimum Lidarr Version:** ${{ inputs.minimum_lidarr_version }} **Commit:** ${{ inputs.git_commit }} EOL # Debug: Show what was generated echo "Generated release notes:" cat release_notes.md - name: Upload Release Notes uses: actions/upload-artifact@v4 with: name: ${{ steps.create_notes.outputs.release_notes_id }} path: release_notes.md retention-days: 1 ================================================ FILE: .gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ _plugins/ _tests/ x64/ x86/ [Ww][Ii][Nn]32/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Oo]ut/ [Ll]og/ [Ll]ogs/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUnit *.VisualState.xml TestResult.xml nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # ASP.NET Scaffolding ScaffoldingReadMe.txt # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool coverage*.json coverage*.xml coverage*.info # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # NuGet Symbol Packages *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- [Bb]ackup.rdl *- [Bb]ackup ([0-9]).rdl *- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ # Fody - auto-generated XML schema FodyWeavers.xsd /_plugins/SpotiarrTube /_plugins/Tubifarry /enc_temp_folder/f7b6bdaa49df014838e55dd376c78df /_plugins/UnitTest /enc_temp_folder ================================================ FILE: .gitmodules ================================================ [submodule "Submodules/Lidarr"] path = Submodules/Lidarr url = https://github.com/Lidarr/Lidarr/ ================================================ FILE: CONTRIBUTION.md ================================================ # Contributing to Tubifarry Thank you for your interest in contributing to Tubifarry! This guide will help you set up your development environment and understand our contribution process. Tubifarry is a plugin for Lidarr that extends its functionality. ## Table of Contents - [Prerequisites](#prerequisites) - [Understanding the Architecture](#understanding-the-architecture) - [Automatic Setup (Recommended)](#automatic-setup-recommended) - [Build Automation Explained](#build-automation-explained) - [Manual Setup (Fallback)](#manual-setup-fallback) - [Development Workflow](#development-workflow) - [Contributing Code](#contributing-code) - [Pull Request Guidelines](#pull-request-guidelines) - [Troubleshooting](#troubleshooting) - [Getting Help](#getting-help) ## Prerequisites Before you begin, ensure you have the following installed on your system: ### Required Software - **Visual Studio 2022** or higher (Community edition is free) - Download: https://visualstudio.com/downloads/ - Must include .NET 8 SDK (VS 2022 V17.0+) - **Git** - Version control system - **Node.js 20.x** - JavaScript runtime for frontend development - Download: https://nodejs.org/ - ⚠️ **Important**: Versions 18.x, 16.x, or 21.x will NOT work - **Yarn** - Package manager for Node.js dependencies - Included with Node 20+ (enable with `corepack enable`) - For other versions: `npm i -g corepack` ### Optional but Recommended - **Rider** by JetBrains (alternative to Visual Studio) - **VS Code** or similar text editor for quick edits ### System Requirements - **Windows**: Windows 10/11 - **Linux**: Any modern distribution - **macOS**: 10.14+ (Mojave or newer) - **RAM**: Minimum 8GB (16GB recommended) - **Storage**: At least 5GB free space for the full build ## Understanding the Architecture Tubifarry is built as a plugin that integrates with Lidarr's core functionality: - **Lidarr Core**: The base music management application (included as a Git submodule) - **Tubifarry Plugin**: Your plugin code that extends Lidarr - **Frontend**: React-based UI (part of Lidarr) - **Backend**: C# .NET 8 code (both Lidarr and Tubifarry) ### Important Notes About the Build Process 1. **Lidarr Submodule**: Tubifarry includes Lidarr as a Git submodule, which means it contains a reference to a specific version of Lidarr 2. **Force Push Warning**: The Lidarr develop branch is regularly rebased and force-pushed. This is normal and expected behavior 3. **Build Order Matters**: You must build Lidarr completely before building Tubifarry 4. **First Build Takes Time**: The initial setup can take 5-10 minutes, but subsequent builds are much faster ## Automatic Setup (Recommended) The project includes build automation that handles most of the setup process automatically. This is the fastest way to get started. ### Step 1: Clone the Repository ```bash git clone https://github.com/TypNull/Tubifarry.git ``` Or if you've forked the repository: ```bash git clone https://github.com/YOUR-USERNAME/Tubifarry.git ``` ### Step 2: Open and Build 1. **Open** `Tubifarry.sln` in Visual Studio 2. **Build the solution** by pressing `Ctrl + Shift + B` (or `Build` → `Build Solution`) ⏳ **Wait patiently** - This first build will take some time depending on your internet connection and PC performance. The build automation will: - Initialize and clone all Git submodules (Lidarr source code) - Install Node.js dependencies via Yarn - Build the Lidarr frontend You can monitor progress in the **Output** window (`View` → `Output`). ⚠️ You will see errors on this build. ### Step 3: Restart Visual Studio After the first build completes: 1. **Close** Visual Studio completely 2. **Reopen** Visual Studio 3. **Open** `Tubifarry.sln` again This ensures Visual Studio properly recognizes all the newly generated files and references. ### Step 4: Build Again 1. **Build the solution** again (`Ctrl + Shift + B`) ⚠️ You may see some errors on this build - **these can be ignored**. 2. **Build once more** (`Ctrl + Shift + B`) The build should now complete successfully. ### Step 5: Configure Startup Project 1. In **Solution Explorer**, navigate to: `Submodules` → `Lidarr` → find `Lidarr.Console` 2. **Right-click** on `Lidarr.Console` 3. Select **"Set as Startup Project"** ### Step 6: Run Press `F5` or click the **Start** button to run Lidarr. --- **That's it!** If everything worked, you're ready to start developing. If you encountered errors, see the [Manual Setup](#manual-setup-fallback) or [Troubleshooting](#troubleshooting) sections below. ## Build Automation Explained The project uses several MSBuild `.targets` files to automate the setup and build process. Understanding these can help with troubleshooting. ### PreBuild.targets - Automatic Initialization This file handles automatic setup before the main build starts: **Submodule Initialization (`InitializeSubmodules` target)** - Checks if the Lidarr submodule exists by looking for `Submodules/Lidarr/src/NzbDrone.Core/Lidarr.Core.csproj` - If missing, automatically runs `git submodule update --init --recursive` **Frontend Build (`LidarrFrontendBuild` target)** - Checks if the Lidarr frontend has been built by looking for `Submodules/Lidarr/_output/UI/` - If missing, automatically runs: - `yarn install` - Downloads all Node.js dependencies - `yarn build` - Compiles the React frontend - This can take several minutes on first run ### ILRepack.targets - Assembly Merging After the build completes, this file merges all plugin dependencies into a single DLL: ### Debug.targets - Automatic Plugin Deployment In Debug configuration, this file automatically deploys your plugin: - After ILRepack finishes, copies the plugin files to Lidarr's plugin directory: - `C:\ProgramData\Lidarr\plugins\AUTHOR\Tubifarry\` - Allows immediate testing without manual file copying ### Build Order The targets execute in this order: 1. `InitializeSubmodules` → Ensures Lidarr source exists 2. `LidarrFrontendBuild` → Ensures frontend is compiled 3. `BeforeBuild` → Standard .NET build preparation 4. `Build` → Compiles Tubifarry code 5. `CopySystemTextJson` → Copies required runtime DLL 6. `ILRepacker` → Merges all assemblies 7. `PostBuild` (Debug only) → Deploys to plugin folder ## Manual Setup (Fallback) If the automatic setup fails or you prefer manual control, follow these steps. ### Step 1: Fork and Clone 1. **Fork the Repository** - Go to https://github.com/TypNull/Tubifarry - Click the "Fork" button in the top right - This creates your own copy of the repository 2. **Clone Your Fork** ```bash git clone https://github.com/YOUR-USERNAME/Tubifarry.git cd Tubifarry ``` ### Step 2: Initialize Git Submodules The Tubifarry repository includes Lidarr as a submodule. You need to initialize and download it: ```bash git submodule update --init --recursive ``` This command downloads the Lidarr source code into the submodule directory. This may take a few minutes depending on your internet connection. ### Step 3: Verify Submodule Check that the Lidarr submodule was properly initialized: ```bash cd Submodules/Lidarr/ git status ``` You should see the Lidarr repository files. ### Step 4: Build Lidarr Frontend The frontend must be built before proceeding to the backend. 1. **Navigate to the Lidarr submodule directory** ```bash cd Submodules/Lidarr/ ``` 2. **Install Node dependencies** ```bash yarn install ``` This downloads all required JavaScript packages. First time takes 2-5 minutes. 3. **Build the frontend** (or start the watcher for development) ```bash yarn build ``` Or for active development with hot-reload: ```bash yarn start ``` **Important**: If using `yarn start`, keep this terminal window open. ### Step 5: Build Lidarr Backend 1. **Open the Lidarr solution in Visual Studio** - Navigate to the Lidarr submodule directory - Open `Lidarr.sln` in Visual Studio 2022 2. **Configure the startup project** - Right-click on `Lidarr.Console` in Solution Explorer - Select "Set as Startup Project" 3. **Build the solution** - Click `Build` → `Build Solution` (or press `Ctrl+Shift+B`) - Wait for the build to complete (first build takes a bit time) - Watch the Output window for any errors ### Step 6: Build Tubifarry Plugin Now that Lidarr is fully built, you can build the Tubifarry plugin. 1. **Navigate back to the Tubifarry root directory** ```bash cd [path-to-Tubifarry-root] ``` 2. **Open the Tubifarry solution** - Open `Tubifarry.sln` in Visual Studio (in a new instance or after closing Lidarr solution) 3. **Wait for dependencies to load** - Visual Studio will restore NuGet packages automatically - Wait until the status bar shows "Ready" 4. **Build the Tubifarry solution** - Click `Build` → `Build Solution` - The plugin will automatically copy to the correct directory on Windows in Debug mode ### Step 7: Manual Plugin Deployment (if needed) If the automatic deployment doesn't work, manually copy the plugin: 1. Find the built plugin at: `Tubifarry/bin/Debug/net8.0/Lidarr.Plugin.Tubifarry.dll` 2. Copy to: `C:\ProgramData\Lidarr\plugins\AUTHOR\Tubifarry\` 3. Also copy the `.pdb` and `.deps.json` files if they exist ## Development Workflow ### Daily Development Process Once you've completed the initial setup, your typical workflow will be: 1. **Make your changes** to Tubifarry code in Visual Studio 2. **Build Tubifarry** to test your changes (`Ctrl + Shift + B`) 3. **Run and test** - Ensure `Lidarr.Console` is set as the startup project - Press `F5` to start debugging - Lidarr will start with your plugin loaded - Access the UI at http://localhost:8686 ### Hot Reload (Optional) For faster frontend iteration: 1. Keep `yarn start` running in the Lidarr submodule directory 2. Frontend changes will automatically refresh in the browser ## Contributing Code ### Before You Start - **Check existing issues**: Look at [GitHub Issues](https://github.com/TypNull/Tubifarry/issues) to see if your feature/bug is already being worked on - **Create an issue**: If your idea isn't already tracked, create a new issue to discuss it - **Ask questions**: Join our Discord or comment on the issue if you need clarification ### Code Guidelines 1. **Code Style** - Use 4 spaces for indentation (not tabs) - Follow C# naming conventions - Keep methods focused and reasonably sized - Add XML documentation comments for public APIs 2. **Commit Guidelines** - Make meaningful commits with clear messages - Use \*nix line endings (LF, not CRLF) - Commit message format: - `New: Add Spotify playlist import feature` - `Fixed: YouTube download timeout issue` - `Improved: Error handling in Slskd client` 3. **Code Quality** - Test your changes thoroughly - Handle errors gracefully - Add logging for debugging purposes - Don't leave commented-out code ### Branching Strategy 1. **Create a feature branch** from `develop` (not from `master`) ```bash git checkout develop git pull origin develop git checkout -b feature/your-feature-name ``` 2. **Use descriptive branch names** - ✅ Good: `feature/spotify-auth`, `fix/youtube-timeout`, `improve/error-messages` - ❌ Bad: `patch`, `updates`, `my-branch` 3. **Keep your branch updated** ```bash git checkout develop git pull origin develop git checkout feature/your-feature-name git rebase develop ``` ## Pull Request Guidelines ### Before Submitting - [ ] Code builds without errors - [ ] You've tested the changes locally - [ ] Code follows the style guidelines - [ ] Commits are clean and well-organized (consider squashing if needed) - [ ] Branch is up to date with `develop` ### Creating a Pull Request 1. **Push your branch** to your fork ```bash git push origin feature/your-feature-name ``` 2. **Open a Pull Request** on GitHub - Go to your fork on GitHub - Click "Compare & pull request" - Target the `develop` branch (not `master`) 3. **Fill out the PR template** - Describe what your PR does - Reference any related issues (e.g., "Fixes #123") - Add screenshots if it's a UI change - List any breaking changes 4. **Respond to feedback** - Maintainers may request changes - Make the requested updates in your branch - Push the changes (they'll automatically update the PR) ### PR Review Process - We aim to review PRs within a few days - If it's been longer, feel free to ping us on Discord - Be patient and respectful during code review - All PRs require approval before merging ### Important Rules - ⛔ **Never** make PRs to `master` - they will be closed - ⛔ **Don't** merge `develop` into your feature branch - use rebase instead - ✅ **Do** create one PR per feature/bugfix - ✅ **Do** ask questions if you're unsure ## Troubleshooting ### Common Build Errors #### "Could not load file or assembly 'Lidarr.Core'" **Cause**: Lidarr backend wasn't fully built before building Tubifarry. **Solution**: 1. Ensure Lidarr frontend is built (`Submodules/Lidarr/_output/UI/` exists) 2. Open and build `Lidarr.sln` first 3. Then build Tubifarry #### "Node version not supported" or "Yarn command not found" **Cause**: Wrong Node.js version or Yarn not enabled. **Solution**: ```bash # Check Node version node --version # Should be 20.x # Enable Yarn corepack enable # Verify Yarn is available yarn --version ``` #### Frontend changes not appearing **Cause**: Frontend not rebuilt or browser cache. **Solution**: 1. Navigate to Lidarr submodule directory 2. Run `yarn build` (or `yarn start` for development) 3. Refresh your browser with `Ctrl+F5` (hard refresh) #### "Submodule not initialized" or empty Lidarr directory **Cause**: Git submodules weren't initialized. **Solution**: ```bash git submodule update --init --recursive ``` #### Build succeeds but plugin doesn't appear in Lidarr **Cause**: Plugin DLL not copied to correct location. **Solution**: 1. Check if you're building in **Debug** configuration (automatic copy only works in Debug) 2. Check your build output directory: `Tubifarry/bin/Debug/net8.0/` 3. Manually copy plugin files to: `C:\ProgramData\Lidarr\plugins\AUTHOR\Tubifarry\` 4. Restart Lidarr 5. Check Lidarr logs for plugin loading errors #### ILRepack errors **Cause**: Missing or incompatible assemblies. **Solution**: 1. Clean the solution (`Build` → `Clean Solution`) 2. Delete the `bin` and `obj` folders in the Tubifarry project 3. Rebuild ### Performance Issues #### First build is very slow This is normal! The first build includes: - Downloading all NuGet packages - Downloading all Node packages - Building the frontend (webpack compilation) - Building everything from scratch Subsequent builds will be much faster (usually under 1 minute). #### Visual Studio is slow or freezes **Solutions**: - Close unnecessary programs to free up RAM - Disable unnecessary VS extensions - Clear VS cache: `Tools` → `Options` → `Projects and Solutions` → `Build and Run` → check "Only build startup projects" - Consider using Rider instead (lighter weight) ### Getting More Help If you're still stuck: 1. **Check existing issues**: https://github.com/TypNull/Tubifarry/issues 2. **Search Discord**: Past questions may have been answered 3. **Ask on Discord**: Include: - What you're trying to do - What error you're seeing - What you've already tried - Your OS and software versions ## Getting Help ### Support Channels - **GitHub Issues**: For bug reports and feature requests - **Servarr Discord**: For development questions and discussions ### Tips for Getting Help When asking for help, please include: 1. **What you're trying to accomplish** 2. **What error you're encountering** (exact error message) 3. **What you've already tried** 4. **Your environment**: - Operating System - Visual Studio version - .NET SDK version (`dotnet --version`) - Node.js version (`node --version`) --- ## Quick Reference ### First-Time Setup Checklist - [ ] Install Visual Studio 2022 with .NET 8 SDK - [ ] Install Git - [ ] Install Node.js 20.x - [ ] Enable Yarn (`corepack enable`) - [ ] Fork Tubifarry repository - [ ] Clone your fork - [ ] Initialize submodules (`git submodule update --init --recursive`) - [ ] Build Lidarr frontend (`cd Submodules/Lidarr && yarn install && yarn build`) - [ ] Build Lidarr backend (open `Lidarr.sln`, build) - [ ] Build Tubifarry plugin (open `Tubifarry.sln`, build) - [ ] Test by running Lidarr.Console ### Before Submitting a PR - [ ] Code builds successfully - [ ] Changes tested locally - [ ] Commits are clean with good messages - [ ] Branch rebased with latest `develop` - [ ] PR targets `develop` (not `master`) - [ ] PR description is complete --- Thank you for contributing to Tubifarry! Your contributions help make music management better for everyone. ================================================ FILE: Directory.Build.props ================================================ AnyCPU true $(MSBuildThisFileDirectory) Library Test Exe Exe Exe Exe Update false true true true true Release $(LidarrRootDir)_temp\obj\$(MSBuildProjectName)\ $(LidarrRootDir)_temp\obj\$(MSBuildProjectName)\$(Configuration)\ $(LidarrRootDir)_temp\bin\$(Configuration)\$(MSBuildProjectName)\ $(LidarrRootDir)_output\ $(LidarrRootDir)_plugins\$(TargetFramework)\$(MSBuildProjectName) $(LidarrRootDir)_tests\ $(LidarrRootDir)_output\Lidarr.Update\ $([MSBuild]::MakeRelative('$(MSBuildProjectDirectory)', '$(BaseIntermediateOutputPath)')) $([MSBuild]::MakeRelative('$(MSBuildProjectDirectory)', '$(IntermediateOutputPath)')) $([MSBuild]::MakeRelative('$(MSBuildProjectDirectory)', '$(OutputPath)')) true true true false <_Parameter1>$(AssemblyConfiguration) false $(MSBuildProjectName.Replace('Lidarr','NzbDrone')) all runtime; build; native; contentfiles; analyzers ================================================ FILE: LICENSE.txt ================================================ MIT License Copyright (c) 2026 TypNull Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: NuGet.config ================================================ ================================================ FILE: README.md ================================================ # Tubifarry for Lidarr 🎶 ![Downloads](https://img.shields.io/github/downloads/TypNull/Tubifarry/total) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/TypNull/Tubifarry) ![GitHub last commit](https://img.shields.io/github/last-commit/TypNull/Tubifarry) ![License](https://img.shields.io/github/license/TypNull/Tubifarry) ![GitHub stars](https://img.shields.io/github/stars/TypNull/Tubifarry) Tubifarry is a plugin for **Lidarr** that adds multiple music sources to your library management. It uses **Spotify's catalog** as an [indexer](https://wiki.servarr.com/en/lidarr/supported#indexers) to search for music, then downloads the actual audio files from **YouTube**. Tubifarry also supports **Slskd**, the Soulseek client, as both an **indexer** and **downloader**, allowing you to tap into the vast music collection available on the Soulseek network. 🛠️ Additionally, Tubifarry supports fetching soundtracks from **Sonarr** (series) and **Radarr** (movies) and adding them to Lidarr using the **Arr-Soundtracks** import list feature. This makes it easy to manage and download soundtracks for your favorite movies and TV shows. 🎬🎵 For further customization, Codec Tinker lets you convert audio files between formats using FFmpeg, helping you optimize your library.⚙️ > **Note**: Some details in this documentation may vary from the current implementation. --- ## Table of Contents 📑 1. [Installation 🚀](#installation-) 2. [Soulseek (Slskd) Setup 🎧](#soulseek-slskd-setup-) 3. [YouTube Downloader Setup 🎥](#youtube-downloader-setup-) 4. [WebClients 📻](#web-clients-) 5. [Fetching Soundtracks 🎬🎵](#fetching-soundtracks-from-sonarr-and-radarr-) 6. [Queue Cleaner 🧹](#queue-cleaner-) 7. [Codec Tinker 🎛️](#codec-tinker-️) 8. [Lyrics Fetcher 📜](#lyrics-fetcher-) 9. [Search Sniper 🏹](#search-sniper-) 10. [Custom Metadata Sources 🧩](#custom-metadata-sources-) 11. [Similar Artists 🧷](#similar-artists-) 12. [Troubleshooting 🛠️](#troubleshooting-%EF%B8%8F) ---- ## Installation 🚀 Follow the steps below to get started. - In Lidarr, go to `System -> Plugins`. - Paste `https://github.com/TypNull/Tubifarry` into the GitHub URL box and click **Install**. --- ### Soulseek (Slskd) Setup 🎧 Tubifarry supports **Slskd**, the Soulseek client, as both an **indexer** and **downloader**. Follow the steps below to configure it. #### **Setting Up the Soulseek Indexer**: 1. Navigate to `Settings -> Indexers` and click **Add**. 2. Select `Slskd` from the list of indexers. 3. Configure the following settings: - **URL**: The URL of your Slskd instance (e.g., `http://localhost:5030`). - **API Key**: The API key for your Slskd instance (found in Slskd's settings under 'Options'). - **Include Only Audio Files**: Enable to filter search results to audio files only. #### **Setting Up the Soulseek Download Client**: 1. Go to `Settings -> Download Clients` and click **Add**. 2. Select `Slskd` from the list of download clients. 3. The download path is fetched from slskd, if it does not match use `Remote Path` settings. --- ### YouTube Downloader Setup 🎥 > #### YouTube Bot Detection ⚠️ > YouTube actively detects and blocks automated downloaders. To bypass this, configure the Trusted Session Generator and provide cookie authentication (see setup steps below). The YouTube downloader extracts audio from YouTube and converts them to audio files using FFmpeg. #### **Configure the Indexer**: 1. Navigate to `Settings -> Indexers` and click **Add**. 2. In the modal, select `Tubifarry` (located under **Other** at the bottom). #### **Setting Up the YouTube Download Client**: 1. Go to `Settings -> Download Clients` and click **Add**. 2. Select `Youtube` from the list of download clients. 3. Set the download path and adjust other settings as needed. 4. **Optional**: If using FFmpeg, ensure the FFmpeg path is correctly configured. #### **FFmpeg and Audio Conversion**: 1. **FFmpeg**: Required to extract audio from YouTube. The plugin will attempt to download FFmpeg automatically if not found. Without FFmpeg, files will be downloaded in their original format, which Lidarr may cannot properly import. - Ensure FFmpeg is in your system PATH or specify its location in settings - Used for extracting audio tracks and converting between formats 2. **Audio Quality**: YouTube provides audio at various bitrates: - Standard quality: 128kbps AAC (free users) - High quality: 256kbps AAC (YouTube Premium required) - The plugin can convert to other formats (MP3, Opus) using FFmpeg --- ### Web Clients 📻 Tubifarry supports multiple web clients. These are web services that provide music. Some work better than others and Tubifarry is not responsible for the uptime or stability of these services. ##### Supported Clients - **Lucida** - A music downloading service that supports multiple sources. - **DABmusic** - A high-resolution audio streaming platform. - **T2Tunes** - A music downloading service that supports AmazonMusic - **Subsonic** - A music streaming API standard with broad compatibility All clients share the same base architecture, making it relatively straightforward to add new ones. The Subsonic Indexer and Client is a generic client, making it possible for any online service to connect with it. The Subsonic specifications are documented on the [API page](https://www.subsonic.org/pages/api.jsp). If you have a suggestion to add a web client and the service does not want to support Subsonic as a generic indexer, please open a feature request. --- ### Fetching Soundtracks from Sonarr and Radarr 🎬🎵 Tubifarry also supports fetching soundtracks from **Sonarr** (for TV series) and **Radarr** (for movies) and adding them to Lidarr using the **Arr-Soundtracks** import list feature. This allows you to easily manage and download soundtracks for your favorite movies and TV shows. To enable this feature: 1. **Set Up the Import List**: - Navigate to `Settings -> Import Lists` in Lidarr. - Add a new import list and select the option for **Arr-Soundtracks**. - Configure the settings to match your Sonarr and Radarr instances. - Provide a cache path to store responses from MusicBrainz for faster lookups. 2. **Enjoy Soundtracks**: - Once configured, Tubifarry will automatically fetch soundtracks from your Sonarr and Radarr libraries and add them to Lidarr for download and management. --- ### Queue Cleaner 🧹 The **Queue Cleaner** automatically handles downloads that fail to import into your library. When Lidarr can't import a download (due to missing tracks, incorrect metadata, etc.), Queue Cleaner can rename files based on their embedded tags, retry the import, blocklist the release, or remove the files. 1. **Key Options**: - *Blocklist*: Choose to remove, blocklist, or both for failed imports. - *Rename*: Automatically rename album folders and tracks using available metadata. - *Clean Imports*: Decide when to clean—when tracks are missing, metadata is incomplete, or always. - *Retry Finding Release*: Automatically retry searching for a release if the import fails. 2. **How to Enable**: - Navigate to `Settings -> Connect` in Lidarr. - Add a new connection and select the **Queue Cleaner**. - Configure the settings to match your needs. --- ### Codec Tinker 🎛️ **Codec Tinker** automatically converts audio files between formats using FFmpeg when they're imported into your library. You can set up rules to convert specific formats (e.g., convert all WAV files to FLAC, or convert high-bitrate AAC to MP3). Note: Lossy formats (MP3, AAC) cannot be converted to lossless formats (FLAC, WAV) as quality cannot be restored. #### How to Enable Codec Tinker 1. Go to `Settings > Metadata` in Lidarr. 2. Open the **Codec Tinker** MetadataConsumer. 3. Toggle the switch to enable the feature. #### How to Use Codec Tinker 1. **Set Default Conversion Settings** - **Target Format**: Choose the default format for conversions (e.g., FLAC, Opus, MP3). - **Custom Conversion Rules**: Define rules like `wav -> flac`, `AAC>=256k -> MP3:300k` or `all -> alac` for more specific conversions. - **Custom Conversion Rules On Artists**: Define tags like `opus-192` for one specific conversion on all albums of an artist. **Note**: Lossy formats (e.g., MP3, AAC) cannot be converted to non-lossy formats (e.g., FLAC, WAV). 2. **Enable Format-Specific Conversion** Toggle checkboxes or use custom rules to enable conversion for specific formats: - **Convert MP3**, **Convert FLAC**, etc. --- ### Lyrics Fetcher 📜 **Lyrics Fetcher** automatically downloads lyrics for your music files. It fetches synchronized lyrics from LRCLIB and plain lyrics from Genius. Lyrics can be saved as separate .lrc files and embedded directly into the audio files' metadata. #### How to Enable Lyrics Fetcher 1. Go to `Settings > Metadata` in Lidarr. 2. Open the **Lyrics Fetcher** MetadataConsumer. 3. Toggle the switch to enable the feature. #### How to Use Lyrics Fetcher You can configure the following options: - **Create LRC Files**: Enables creating external `.lrc` files that contain time-synced lyrics. - **Embed Lyrics in Audio Files**: Instead of (or in addition to) creating separate LRC files, this option embeds the lyrics directly into the audio file's. - **Overwrite Existing LRC Files**: When enabled, this will replace any existing LRC files with newly downloaded ones. --- ### Search Sniper 🏹 **Search Sniper** automatically triggers searches for missing albums in your wanted list. Instead of searching for everything at once, which can overload indexers, it randomly selects a few albums from your wanted list at regular intervals and searches for them. It keeps track of what has been searched recently to avoid repeating searches too often. Search Sniper can be triggered manually from the Tasks tab. #### How to Enable Search Sniper 1. Go to `Settings > Metadata` in Lidarr. 2. Open the **Search Sniper** option. 3. Configure the following options: - **Picks Per Interval**: How many items to search each cycle - **Min Refresh Interval**: How often to run searches - **Cache Type**: Memory or Permanent - **Cache Retention Time**: Days to keep cache - **Pause When Queued**: Stop when queue reaches this number - **Search Options**: Enable at least one - Missing albums, Missing tracks, or Cutoff not met --- ### Custom Metadata Sources 🧩 Tubifarry can fetch metadata from additional sources beyond MusicBrainz, including **Discogs**, **Deezer**, and **Last.fm**. These sources can provide additional artist information, album details, and cover art when MusicBrainz data is incomplete. The MetaMix feature intelligently combines data from multiple sources to create more complete metadata profiles. #### How to Enable Individual Metadata Sources 1. Go to `Settings > Metadata` in Lidarr. 2. Open a specific **Metadata Source**. 3. Toggle the switch to enable the feature. 4. Configure the required settings: - **User Agent**: Set a custom identifier that follows the format "Name/Version" to help the metadata service identify your requests properly. - **API Key**: Enter your personal access token or API key for the service. - **Caching Method**: Choose between: - **Memory Caching**: Faster but less persistent (only recommended if your system has been running stably for 5+ days) - **Permanent Caching**: More reliable but requires disk storage - **Cache Directory**: If using Permanent caching, specify a folder where metadata can be stored to reduce API calls. #### How to Enable Multiple Metadata Sources MetaMix is an advanced feature that intelligently combines metadata from multiple sources to create more complete artist profiles. It can fill gaps in one source with information from another, resulting in a more comprehensive music library. 1. Go to `Settings > Metadata` in Lidarr. 2. Open the **MetaMix** settings. 3. Configure the following options: - **Priority Rules**: Establish a hierarchy among your metadata sources. For example, set MusicBrainz as primary and Discogs as secondary. Lower numbers indicate higher priority. - **Dynamic Threshold**: Controls how aggressively MetaMix switches between sources: - Higher values make MetaMix more willing to use lower-priority sources - Lower values make MetaMix stick more closely to your primary source - **Multi-Source Population**: When enabled, missing album information from your primary source will be automatically supplemented with data from secondary sources. The feature currently works best with artists that are properly linked across different metadata systems. Which is typically the case on MusicBrainz. --- ### Similar Artists 🧷 **Similar Artists** lets you discover related artists using Last.fm's recommendation data directly in Lidarr's search. Search for an artist with the `~` prefix and get back a list of similar musicians ready to be added to your library. #### How to Enable Similar Artists 1. Go to `Settings > Metadata` in Lidarr. 2. Enable these three metadata sources: - **Similar Artists** - Enter your Last.fm API key - **Lidarr Default** - Required to handle normal searches - **MetaMix** - Required to coordinate the search 3. Optional: Adjust result limit, enable image fetching, and configure caching. **Examples:** - `similar:Pink Floyd` - `~20244d07-534f-4eff-b4d4-930878889970` --- ## Troubleshooting 🛠️ - **Slskd Download Path Permissions**: Ensure Lidarr has read/write access to the Slskd download path. Verify folder permissions and confirm the user running Lidarr has the necessary access. For Docker setups, double-check that the volume is correctly mounted and permissions are properly configured. - **FFmpeg Issues (Optional)**: If you're using FFmpeg and songs fail to process, ensure FFmpeg is installed correctly and accessible in your system's PATH. If issues persist, try reinstalling FFmpeg or downloading it manually. - **Metadata Issues**: If metadata isn't being added to downloaded files, confirm the files are in a supported format. If using FFmpeg, check that it's extracting audio to compatible formats like AAC embedded in MP4 containers. Review debug logs for further details. - **No Release Found**: If no release is found, YouTube may flag the plugin as a bot. To avoid this and access higher-quality audio, use a combination of cookies and the Trusted Session Generator: 1. Install the **cookies.txt** extension for your browser: - [Chrome](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) - [Firefox](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) 2. Log in to YouTube and save the `cookies.txt` file in a folder accessible by Lidarr. 3. In Lidarr, go to **Indexer and Downloader Settings** and provide the path to the `cookies.txt` file. 4. **Trusted Session Generator**: Creates authentication tokens that mimic regular browser sessions to bypass YouTube's bot detection. - It generates tokens locally, which requires Node.js installed and available in your system's PATH - It can generate tokens using the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider) The combination of cookies and trusted sessions significantly improves success rates when downloading from YouTube, and can help access higher quality audio streams. - **No Lyrics Imported**: To save `.lrc` files (lyric files), navigate to **Media Management > Advanced Settings > Import Extra Files** and add `lrc` to the list of supported file types. This ensures lyric files are imported and saved alongside your music files. - **Unsupported Formats**: Verify custom rules and target formats. --- ## Acknowledgments 🙌 Special thanks to [**trevTV**](https://github.com/TrevTV) for laying the groundwork with his plugins. Additionally, thanks to [**IcySnex**](https://github.com/IcySnex) for providing the YouTube API. 🎉 --- ## Contributing 🤝 If you'd like to contribute to Tubifarry, feel free to open issues or submit pull requests on the [GitHub repository](https://github.com/TypNull/Tubifarry). Your feedback and contributions are highly appreciated! --- ## License 📄 Tubifarry is licensed under the MIT License. See the [LICENSE](https://github.com/TypNull/Tubifarry/blob/master/LICENSE.txt) file for more details. --- Enjoy seamless music downloads with Tubifarry! 🎧 ================================================ FILE: Tubifarry/Blocklisting/BaseBlocklist.cs ================================================ using NzbDrone.Common.Extensions; using NzbDrone.Core.Blocklisting; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; namespace Tubifarry.Blocklisting { public abstract class BaseBlocklist(IBlocklistRepository blocklistRepository) : IBlocklistForProtocol where TProtocol : IDownloadProtocol { private readonly IBlocklistRepository _blocklistRepository = blocklistRepository; public string Protocol => typeof(TProtocol).Name; public bool IsBlocklisted(int artistId, ReleaseInfo release) => _blocklistRepository.BlocklistedByTorrentInfoHash(artistId, release.Guid).Any(b => BaseBlocklist.SameRelease(b, release)); public Blocklist GetBlocklist(DownloadFailedEvent message) => new() { ArtistId = message.ArtistId, AlbumIds = message.AlbumIds, SourceTitle = message.SourceTitle, Quality = message.Quality, Date = DateTime.UtcNow, PublishedDate = DateTime.TryParse(message.Data.GetValueOrDefault("publishedDate") ?? string.Empty, out DateTime publishedDate) ? publishedDate : null, Size = long.Parse(message.Data.GetValueOrDefault("size", "0")), Indexer = message.Data.GetValueOrDefault("indexer"), Protocol = message.Data.GetValueOrDefault("protocol"), Message = message.Message, TorrentInfoHash = message.Data.GetValueOrDefault("guid") }; private static bool SameRelease(Blocklist item, ReleaseInfo release) => release.Guid.IsNotNullOrWhiteSpace() ? release.Guid.Equals(item.TorrentInfoHash) : item.Indexer.Equals(release.Indexer, StringComparison.InvariantCultureIgnoreCase); } } ================================================ FILE: Tubifarry/Blocklisting/Blocklists.cs ================================================ using NzbDrone.Core.Blocklisting; using NzbDrone.Core.Indexers; namespace Tubifarry.Blocklisting { public class YoutubeBlocklist(IBlocklistRepository blocklistRepository) : BaseBlocklist(blocklistRepository) { } public class SoulseekBlocklist(IBlocklistRepository blocklistRepository) : BaseBlocklist(blocklistRepository) { } public class QobuzBlocklist(IBlocklistRepository blocklistRepository) : BaseBlocklist(blocklistRepository) { } public class LucidaBlocklist(IBlocklistRepository blocklistRepository) : BaseBlocklist(blocklistRepository) { } public class SubSonicBlocklist(IBlocklistRepository blocklistRepository) : BaseBlocklist(blocklistRepository) { } } ================================================ FILE: Tubifarry/Core/Model/AlbumData.cs ================================================ using NzbDrone.Core.Parser.Model; using System.Text.RegularExpressions; using Tubifarry.Core.Utilities; namespace Tubifarry.Core.Model { /// /// Contains combined information about an album, search parameters, and search results. /// public partial class AlbumData(string name, string downloadProtocol) { public string? Guid { get; set; } public string IndexerName { get; } = name; // Mixed public string AlbumId { get; set; } = string.Empty; // Properties from AlbumInfo public string AlbumName { get; set; } = string.Empty; public string ArtistName { get; set; } = string.Empty; public string InfoUrl { get; set; } = string.Empty; public string ReleaseDate { get; set; } = string.Empty; public DateTime ReleaseDateTime { get; set; } public string ReleaseDatePrecision { get; set; } = string.Empty; public int TotalTracks { get; set; } public bool ExplicitContent { get; set; } public string CustomString { get; set; } = string.Empty; public string CoverResolution { get; set; } = string.Empty; // Properties from YoutubeSearchResults public int Bitrate { get; set; } public int BitDepth { get; set; } public long Duration { get; set; } // Soulseek public long? Size { get; set; } public int Priotity { get; set; } public List? ExtraInfo { get; set; } public string DownloadProtocol { get; set; } = downloadProtocol; // Not used public AudioFormat Codec { get; set; } = AudioFormat.AAC; /// /// Converts AlbumData into a ReleaseInfo object. /// public ReleaseInfo ToReleaseInfo() => new() { Guid = Guid ?? $"{IndexerName}-{AlbumId}-{Codec}-{Bitrate}-{BitDepth}", Artist = ArtistName, Album = AlbumName, DownloadUrl = AlbumId, InfoUrl = InfoUrl, PublishDate = ReleaseDateTime == DateTime.MinValue ? DateTime.UtcNow : ReleaseDateTime, DownloadProtocol = DownloadProtocol, Title = ConstructTitle(), Codec = Codec.ToString(), Resolution = CoverResolution, Source = CustomString, Container = Bitrate.ToString(), Size = Size ?? (Duration > 0 ? Duration : TotalTracks * 300) * Bitrate * 1000 / 8 }; /// /// Parses the release date based on the precision. /// public void ParseReleaseDate() => ReleaseDateTime = ReleaseDatePrecision switch { "year" => new DateTime(int.Parse(ReleaseDate), 1, 1), "month" => DateTime.ParseExact(ReleaseDate, "yyyy-MM", System.Globalization.CultureInfo.InvariantCulture), "day" => DateTime.ParseExact(ReleaseDate, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), _ => throw new FormatException($"Unsupported release_date_precision: {ReleaseDatePrecision}"), }; /// /// Constructs a title string for the album in a format optimized for parsing. /// /// A formatted title string. private string ConstructTitle() { string normalizedAlbumName = NormalizeAlbumName(AlbumName); string title = $"{ArtistName} - {normalizedAlbumName}"; if (ReleaseDateTime != DateTime.MinValue) title += $" ({ReleaseDateTime.Year})"; if (ExplicitContent) title += " [Explicit]"; int calculatedBitrate = Bitrate; if (calculatedBitrate <= 0 && Size.HasValue && Duration > 0) calculatedBitrate = (int)(Size.Value * 8 / (Duration * 1000)); if (AudioFormatHelper.IsLossyFormat(Codec) && calculatedBitrate != 0) title += $" [{Codec} {calculatedBitrate}kbps]"; else if (!AudioFormatHelper.IsLossyFormat(Codec) && BitDepth != 0) title += $" [{Codec} {BitDepth}bit]"; else title += $" [{Codec}]"; if (ExtraInfo?.Count > 0) title += string.Concat(ExtraInfo.Where(info => !string.IsNullOrEmpty(info)).Select(info => $" [{info}]")); title += " [WEB]"; return title; } /// /// Normalizes the album name to handle featuring artists and other parentheses. /// /// The album name to normalize. /// The normalized album name. private static string NormalizeAlbumName(string albumName) { if (FeatRegex().IsMatch(albumName)) // TODO ISMatch vs Match { Match match = FeatRegex().Match(albumName); string featuringArtist = albumName[(match.Index + match.Length)..].Trim(); albumName = $"{albumName[..match.Index].Trim()} (feat. {featuringArtist})"; } return FeatReplaceRegex().Replace(albumName, match => $"{{{match.Value.Trim('(', ')')}}}"); } [GeneratedRegex(@"(?i)\b(feat\.|ft\.|featuring)\b", RegexOptions.IgnoreCase, "de-DE")] private static partial Regex FeatRegex(); [GeneratedRegex(@"\((?!feat\.)[^)]*\)")] private static partial Regex FeatReplaceRegex(); } } ================================================ FILE: Tubifarry/Core/Model/ApiCircuitBreaker.cs ================================================ using System.Collections.Concurrent; using System.Runtime.CompilerServices; namespace Tubifarry.Core.Model { public interface ICircuitBreaker { bool IsOpen { get; } void RecordSuccess(); void RecordFailure(); void Reset(); } public class ApiCircuitBreaker(int failureThreshold = 5, int resetTimeoutMinutes = 5) : ICircuitBreaker { private int _failureCount; private DateTime _lastFailure = DateTime.MinValue; private readonly int _failureThreshold = failureThreshold; private readonly TimeSpan _resetTimeout = TimeSpan.FromMinutes(resetTimeoutMinutes); private readonly object _lock = new(); public bool IsOpen { get { lock (_lock) { if (_failureCount >= _failureThreshold) { if (DateTime.UtcNow - _lastFailure > _resetTimeout) { Reset(); return false; } return true; } return false; } } } public void RecordSuccess() { lock (_lock) { _failureCount = Math.Max(0, _failureCount - 1); } } public void RecordFailure() { lock (_lock) { _failureCount++; _lastFailure = DateTime.UtcNow; } } public void Reset() { lock (_lock) { _failureCount = 0; } } } public static class CircuitBreakerFactory { private static readonly ConditionalWeakTable _typeBreakers = []; private static readonly ConcurrentDictionary> _namedBreakers = new(); private static readonly object _cleanupLock = new(); private static DateTime _lastCleanup = DateTime.UtcNow; private static readonly TimeSpan _cleanupInterval = TimeSpan.FromMinutes(15); /// /// Gets a circuit breaker for a specific type. /// public static ICircuitBreaker GetBreaker() => GetBreaker(typeof(T)); /// /// Gets a circuit breaker for a specific object. /// public static ICircuitBreaker GetBreaker(object obj) => GetBreaker(obj.GetType()); /// /// Gets a circuit breaker for a specific type. /// public static ICircuitBreaker GetBreaker(Type type) { if (!_typeBreakers.TryGetValue(type, out ICircuitBreaker? breaker)) { breaker = new ApiCircuitBreaker(); _typeBreakers.Add(type, breaker); } return breaker; } /// /// Gets a circuit breaker by name. /// public static ICircuitBreaker GetBreaker(string name) { CleanupIfNeeded(); if (_namedBreakers.TryGetValue(name, out WeakReference? weakRef) && weakRef.TryGetTarget(out ICircuitBreaker? breaker)) return breaker; breaker = new ApiCircuitBreaker(); _namedBreakers[name] = new WeakReference(breaker); return breaker; } /// /// Gets a circuit breaker with custom configuration. /// public static ICircuitBreaker GetCustomBreaker(int failureThreshold, int resetTimeoutMinutes) { Type type = typeof(T); if (!_typeBreakers.TryGetValue(type, out ICircuitBreaker? breaker)) { breaker = new ApiCircuitBreaker(failureThreshold, resetTimeoutMinutes); _typeBreakers.Add(type, breaker); } return breaker; } private static void CleanupIfNeeded() { lock (_cleanupLock) { if (DateTime.UtcNow - _lastCleanup > _cleanupInterval) { CleanupNamedBreakers(); _lastCleanup = DateTime.UtcNow; } } } private static void CleanupNamedBreakers() { foreach (KeyValuePair> kvp in _namedBreakers) { if (!kvp.Value.TryGetTarget(out _)) _namedBreakers.TryRemove(kvp.Key, out _); } } } } ================================================ FILE: Tubifarry/Core/Model/AudioMetadataHandler.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Music; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using Xabe.FFmpeg; using Xabe.FFmpeg.Downloader; namespace Tubifarry.Core.Model { internal class AudioMetadataHandler { private readonly Logger? _logger; private static bool? _isFFmpegInstalled = null; public string TrackPath { get; private set; } public Lyric? Lyric { get; set; } public byte[]? AlbumCover { get; set; } public bool UseID3v2_3 { get; set; } public AudioMetadataHandler(string originalPath) { TrackPath = originalPath; _logger = NzbDroneLogger.GetLogger(this); } /// /// Base codec parameters that don't change with bitrate settings /// private static readonly Dictionary BaseConversionParameters = new() { { AudioFormat.AAC, new[] { "-codec:a aac", "-movflags +faststart", "-aac_coder twoloop" } }, { AudioFormat.MP3, new[] { "-codec:a libmp3lame" } }, { AudioFormat.Opus, new[] { "-codec:a libopus", "-vbr on", "-application audio", "-vn" } }, { AudioFormat.Vorbis, new[] { "-codec:a libvorbis" } }, { AudioFormat.FLAC, new[] { "-codec:a flac", "-compression_level 8" } }, { AudioFormat.ALAC, new[] { "-codec:a alac" } }, { AudioFormat.WAV, new[] { "-codec:a pcm_s16le", "-ar 44100" } }, { AudioFormat.MP4, new[] { "-codec:a aac", "-movflags +faststart", "-aac_coder twoloop" } }, { AudioFormat.AIFF, new[] { "-codec:a pcm_s16be" } }, { AudioFormat.OGG, new[] { "-codec:a libvorbis" } }, { AudioFormat.AMR, new[] { "-codec:a libopencore_amrnb", "-ar 8000" } }, { AudioFormat.WMA, new[] { "-codec:a wmav2" } } }; /// /// Format-specific bitrate/quality parameter templates /// private static readonly Dictionary> QualityParameters = new() { { AudioFormat.AAC, bitrate => bitrate < 256 ? [$"-b:a {bitrate}k"] : ["-q:a 2"] // 2 is highest quality for AAC }, { AudioFormat.MP3, bitrate => { int qualityLevel = bitrate switch { >= 220 => 0, // V0 (~220-260kbps avg) >= 190 => 1, // V1 (~190-250kbps) >= 170 => 2, // V2 (~170-210kbps) >= 150 => 3, // V3 (~150-195kbps) >= 130 => 4, // V4 (~130-175kbps) >= 115 => 5, // V5 (~115-155kbps) >= 100 => 6, // V6 (~100-140kbps) >= 85 => 7, // V7 (~85-125kbps) >= 65 => 8, // V8 (~65-105kbps) _ => 9 // V9 (~45-85kbps) }; return [$"-q:a {qualityLevel}"]; } }, { AudioFormat.Opus, bitrate => [$"-b:a {bitrate}k", "-compression_level 10"] }, { AudioFormat.Vorbis, bitrate => [$"-q:a {AudioFormatHelper.MapBitrateToVorbisQuality(bitrate)}"] }, { AudioFormat.MP4, bitrate => [$"-b:a {bitrate}k"] }, { AudioFormat.OGG, bitrate => [$"-q:a {AudioFormatHelper.MapBitrateToVorbisQuality(bitrate)}"] }, { AudioFormat.AMR, bitrate => [$"-ab {bitrate}k"]}, { AudioFormat.WMA, bitrate => [$"-b:a {bitrate}k"]} }; private static readonly Dictionary> CBRQualityParameters = new() { { AudioFormat.MP3, bitrate => ["-b:a", $"{bitrate}k"] }, { AudioFormat.AAC, bitrate => ["-b:a", $"{bitrate}k"] }, { AudioFormat.Opus, bitrate => ["-b:a", $"{bitrate}k", "-vbr", "off"] }, { AudioFormat.MP4, bitrate => ["-b:a", $"{bitrate}k"] }, { AudioFormat.AMR, bitrate => ["-ab", $"{bitrate}k"] }, { AudioFormat.WMA, bitrate => ["-b:a", $"{bitrate}k"] } }; private static readonly Dictionary> BitDepthParameters = new() { { AudioFormat.FLAC, bitDepth => bitDepth switch { 16 => ["-sample_fmt", "s16"], 24 => ["-sample_fmt", "s32", "-bits_per_raw_sample", "24"], 32 => ["-sample_fmt", "s32"], _ => [] } }, { AudioFormat.WAV, bitDepth => bitDepth switch { 16 => ["-codec:a", "pcm_s16le"], 24 => ["-codec:a", "pcm_s24le"], 32 => ["-codec:a", "pcm_s32le"], _ => [] } }, { AudioFormat.AIFF, bitDepth => bitDepth switch { 16 => ["-codec:a", "pcm_s16be"], 24 => ["-codec:a", "pcm_s24be"], 32 => ["-codec:a", "pcm_s32be"], _ => [] } } }; private static readonly string[] ExtractionParameters = [ "-codec:a copy", "-vn", "-movflags +faststart" ]; private static readonly string[] VideoFormats = [ "matroska", "webm", "mov", "mp4", "m4a", "avi", "asf", "wmv", "wma", "flv", "f4v", "3gp", "3g2", "mxf", "ts", "m2ts" ]; private static readonly HashSet CoverArtCodecs = new(StringComparer.OrdinalIgnoreCase) { "mjpeg", "png", "bmp", "gif", "webp", "jpeg", "jpg", "tiff", "tif" }; private async Task TryExtractCoverArtAsync() { try { using TagLib.File file = TagLib.File.Create(TrackPath); byte[]? data = file.Tag.Pictures?.FirstOrDefault()?.Data?.Data; if (data?.Length > 0) return data; } catch { } try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(TrackPath); IVideoStream? coverStream = mediaInfo.VideoStreams .FirstOrDefault(vs => CoverArtCodecs.Contains(vs.Codec ?? "")); if (coverStream == null) return null; string tempCoverPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.jpg"); try { IConversion conversion = FFmpeg.Conversions.New() .AddParameter($"-i \"{TrackPath}\"") .AddParameter("-an -vcodec copy") .SetOutput(tempCoverPath); await conversion.Start(); if (File.Exists(tempCoverPath)) return await File.ReadAllBytesAsync(tempCoverPath); } finally { if (File.Exists(tempCoverPath)) File.Delete(tempCoverPath); } } catch { } return null; } /// /// Converts audio to the specified format with optional bitrate control. /// /// Target audio format /// Optional target bitrate in kbps /// True if conversion succeeded, false otherwise public async Task TryConvertToFormatAsync(AudioFormat audioFormat, int? targetBitrate = null, int? targetBitDepth = null, bool useCBR = false) { _logger?.Trace($"Converting {Path.GetFileName(TrackPath)} to {audioFormat}" + (targetBitrate.HasValue ? $" at {targetBitrate}kbps" : targetBitDepth.HasValue ? $" at {targetBitDepth}-bit" : "")); if (!CheckFFmpegInstalled()) return false; if (!await TryExtractAudioFromVideoAsync()) return false; _logger?.Trace($"Looking up audio format: {audioFormat}"); if (audioFormat == AudioFormat.Unknown) return true; if (!BaseConversionParameters.ContainsKey(audioFormat)) return false; string finalOutputPath = Path.ChangeExtension(TrackPath, AudioFormatHelper.GetFileExtensionForFormat(audioFormat)); string tempOutputPath = Path.ChangeExtension(TrackPath, $".converted{AudioFormatHelper.GetFileExtensionForFormat(audioFormat)}"); try { if (File.Exists(tempOutputPath)) File.Delete(tempOutputPath); byte[]? preservedCoverArt = AlbumCover?.Length > 0 ? AlbumCover : await TryExtractCoverArtAsync(); IConversion conversion = await FFmpeg.Conversions.FromSnippet.Convert(TrackPath, tempOutputPath); foreach (string parameter in BaseConversionParameters[audioFormat]) conversion.AddParameter(parameter); if (AudioFormatHelper.IsLossyFormat(audioFormat)) { int bitrate = targetBitrate ?? AudioFormatHelper.GetDefaultBitrate(audioFormat); bitrate = AudioFormatHelper.ClampBitrate(audioFormat, bitrate); string[] qualityParams; string mode; if (useCBR && CBRQualityParameters.ContainsKey(audioFormat)) { qualityParams = CBRQualityParameters[audioFormat](bitrate); mode = "CBR"; } else if (QualityParameters.ContainsKey(audioFormat)) { qualityParams = QualityParameters[audioFormat](bitrate); mode = "VBR"; } else { qualityParams = [$"-b:a {bitrate}k"]; mode = "fallback"; } foreach (string param in qualityParams) conversion.AddParameter(param); _logger?.Trace($"Applied {mode} quality parameters for {audioFormat} at {bitrate}kbps: {string.Join(", ", qualityParams)}"); } if (!AudioFormatHelper.IsLossyFormat(audioFormat) && BitDepthParameters.ContainsKey(audioFormat) && targetBitDepth.HasValue) { string[] bitDepthParams = BitDepthParameters[audioFormat](targetBitDepth.Value); foreach (string param in bitDepthParams) conversion.AddParameter(param); _logger?.Trace($"Applied bit depth parameters for {audioFormat}: {targetBitDepth}-bit ({string.Join(", ", bitDepthParams)})"); } _logger?.Trace($"Starting FFmpeg conversion"); await conversion.Start(); if (File.Exists(TrackPath)) File.Delete(TrackPath); File.Move(tempOutputPath, finalOutputPath, true); TrackPath = finalOutputPath; if (preservedCoverArt?.Length > 0) { try { using TagLib.File destFile = TagLib.File.Create(TrackPath); destFile.Tag.Pictures = [new TagLib.Picture(new TagLib.ByteVector(preservedCoverArt)) { Type = TagLib.PictureType.FrontCover, Description = "Album Cover" }]; destFile.Save(); _logger?.Trace("Re-embedded cover art into converted file"); } catch (Exception ex) { _logger?.Warn(ex, "Failed to re-embed cover art after conversion, cover art may be missing"); } } return true; } catch (Exception ex) { _logger?.Error(ex, $"Failed to convert file to {audioFormat}: {TrackPath}"); return false; } } public async Task IsVideoContainerAsync() { try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(TrackPath); bool hasRealVideo = mediaInfo.VideoStreams.Any(vs => !CoverArtCodecs.Contains(vs.Codec ?? "") && !(vs.Duration.TotalSeconds < 1 && vs.Framerate <= 1)); if (hasRealVideo) return true; string probeResult = await Probe.New().Start($"-v error -show_entries format=format_name -of default=noprint_wrappers=1:nokey=1 \"{TrackPath}\""); string formatName = probeResult?.Trim().ToLower() ?? ""; return VideoFormats.Any(container => formatName.Contains(container)); } catch (Exception ex) { _logger?.Error(ex, $"Failed to check file header: {TrackPath}"); return false; } } public async Task TryExtractAudioFromVideoAsync() { if (!CheckFFmpegInstalled()) return false; bool isVideo = await IsVideoContainerAsync(); if (!isVideo) return await EnsureFileExtAsync(); _logger?.Trace($"Extracting audio from video file: {Path.GetFileName(TrackPath)}"); try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(TrackPath); IAudioStream? audioStream = mediaInfo.AudioStreams.FirstOrDefault(); if (audioStream == null) { _logger?.Trace("No audio stream found in video file"); return false; } string codec = audioStream.Codec.ToLower(); string finalOutputPath = Path.ChangeExtension(TrackPath, AudioFormatHelper.GetFileExtensionForCodec(codec)); string tempOutputPath = Path.ChangeExtension(TrackPath, $".extracted{AudioFormatHelper.GetFileExtensionForCodec(codec)}"); if (File.Exists(tempOutputPath)) File.Delete(tempOutputPath); IConversion conversion = await FFmpeg.Conversions.FromSnippet.ExtractAudio(TrackPath, tempOutputPath); foreach (string parameter in ExtractionParameters) conversion.AddParameter(parameter); await conversion.Start(); if (File.Exists(TrackPath)) File.Delete(TrackPath); File.Move(tempOutputPath, finalOutputPath, true); TrackPath = finalOutputPath; await EnsureFileExtAsync(); _logger?.Trace($"Successfully extracted audio to {Path.GetFileName(TrackPath)}"); return true; } catch (Exception ex) { _logger?.Error(ex, $"Failed to extract audio from video: {TrackPath}"); return false; } } /// /// Decrypts an encrypted audio file using FFmpeg with the provided decryption key. /// /// The hex decryption key for the encrypted content. /// The audio codec of the content (e.g., "flac", "opus", "eac3"). /// Cancellation token. /// True if decryption was successful, false otherwise. public async Task TryDecryptAsync(string decryptionKey, string? codec, CancellationToken token = default) { if (string.IsNullOrEmpty(decryptionKey)) return true; if (!CheckFFmpegInstalled()) return false; _logger?.Trace($"Decrypting file: {Path.GetFileName(TrackPath)}"); try { AudioFormat format = AudioFormatHelper.GetAudioFormatFromCodec(codec ?? "aac"); string extension = AudioFormatHelper.GetFileExtensionForFormat(format); string outputPath = Path.ChangeExtension(TrackPath, extension); string tempOutput = Path.ChangeExtension(TrackPath, $".dec{extension}"); if (File.Exists(tempOutput)) File.Delete(tempOutput); IConversion conversion = FFmpeg.Conversions.New() .AddParameter($"-decryption_key {decryptionKey}") .AddParameter($"-i \"{TrackPath}\"") .AddParameter("-c copy") .SetOutput(tempOutput); await conversion.Start(token); if (File.Exists(TrackPath)) File.Delete(TrackPath); File.Move(tempOutput, outputPath, true); TrackPath = outputPath; _logger?.Trace($"Successfully decrypted: {Path.GetFileName(TrackPath)}"); return true; } catch (Exception ex) { _logger?.Error(ex, $"Failed to decrypt file: {TrackPath}"); return false; } } public async Task TryCreateLrcFileAsync(CancellationToken token) { if (Lyric?.SyncedLyrics == null) return false; try { string lrcContent = string.Join(Environment.NewLine, Lyric.SyncedLyrics .Where(lyric => !string.IsNullOrEmpty(lyric.LrcTimestamp) && !string.IsNullOrEmpty(lyric.Line)) .Select(lyric => $"{lyric.LrcTimestamp} {lyric.Line}")); string lrcPath = Path.ChangeExtension(TrackPath, ".lrc"); await File.WriteAllTextAsync(lrcPath, lrcContent, token); _logger?.Trace($"Created LRC file with {Lyric.SyncedLyrics.Count} synced lyrics"); } catch (Exception ex) { _logger?.Error(ex, $"Failed to create LRC file: {Path.ChangeExtension(TrackPath, ".lrc")}"); return false; } return true; } /// /// Ensures the file extension matches the actual audio codec. /// /// True if the file extension is correct or was successfully corrected; otherwise, false. public async Task EnsureFileExtAsync() { try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(TrackPath); string codec = mediaInfo.AudioStreams.FirstOrDefault()?.Codec.ToLower() ?? string.Empty; if (string.IsNullOrEmpty(codec)) return false; string correctExtension = AudioFormatHelper.GetFileExtensionForCodec(codec); string currentExtension = Path.GetExtension(TrackPath); if (!string.Equals(currentExtension, correctExtension, StringComparison.OrdinalIgnoreCase)) { string newPath = Path.ChangeExtension(TrackPath, correctExtension); _logger?.Trace($"Correcting file extension from {currentExtension} to {correctExtension} for codec {codec}"); File.Move(TrackPath, newPath); TrackPath = newPath; } return true; } catch (Exception ex) { _logger?.Error(ex, $"Failed to ensure correct file extension: {TrackPath}"); return false; } } public bool TryEmbedMetadata(Album albumInfo, Track trackInfo) { _logger?.Trace($"Embedding metadata for track: {trackInfo?.Title}"); try { using TagLib.File file = TagLib.File.Create(TrackPath); if (UseID3v2_3) { TagLib.Id3v2.Tag.DefaultVersion = 3; TagLib.Id3v2.Tag.ForceDefaultVersion = true; } else { TagLib.Id3v2.Tag.DefaultVersion = 4; TagLib.Id3v2.Tag.ForceDefaultVersion = false; } if (!string.IsNullOrEmpty(trackInfo?.Title)) file.Tag.Title = trackInfo.Title; if (trackInfo?.AbsoluteTrackNumber > 0) file.Tag.Track = (uint)trackInfo.AbsoluteTrackNumber; if (!string.IsNullOrEmpty(albumInfo?.Title)) file.Tag.Album = albumInfo.Title; if (albumInfo?.ReleaseDate?.Year > 0) file.Tag.Year = (uint)albumInfo.ReleaseDate.Value.Year; if (albumInfo?.AlbumReleases?.Value?.FirstOrDefault()?.TrackCount > 0) file.Tag.TrackCount = (uint)albumInfo.AlbumReleases.Value[0].TrackCount; if (trackInfo?.MediumNumber > 0) file.Tag.Disc = (uint)trackInfo.MediumNumber; string? albumArtistName = albumInfo?.Artist?.Value?.Name; string? trackArtistName = trackInfo?.Artist?.Value?.Name; if (!string.IsNullOrEmpty(albumArtistName)) file.Tag.AlbumArtists = new[] { albumArtistName }; if (!string.IsNullOrEmpty(trackArtistName)) file.Tag.Performers = new[] { trackArtistName }; if (albumInfo?.AlbumReleases?.Value?.FirstOrDefault()?.Label?.Any() == true) file.Tag.Copyright = albumInfo.AlbumReleases.Value[0].Label.FirstOrDefault(); if (albumInfo?.Genres?.Any() == true) { string[] validGenres = albumInfo.Genres.Where(g => !string.IsNullOrEmpty(g)).ToArray(); if (validGenres.Length > 0) file.Tag.Genres = validGenres; } if (trackInfo?.Explicit == true) file.Tag.Comment = "EXPLICIT"; if (!string.IsNullOrEmpty(trackInfo?.ForeignRecordingId) && file.GetTag(TagLib.TagTypes.Id3v2) is TagLib.Id3v2.Tag id3v2Tag) { TagLib.Id3v2.UserTextInformationFrame mbFrame = TagLib.Id3v2.UserTextInformationFrame.Get(id3v2Tag, "MusicBrainz Recording Id", true); mbFrame.Text = [trackInfo.ForeignRecordingId]; } try { if (AlbumCover?.Length > 0) { TagLib.Picture picture = new(new TagLib.ByteVector(AlbumCover)) { Type = TagLib.PictureType.FrontCover, Description = "Album Cover" }; file.Tag.Pictures = [picture]; } } catch (Exception ex) { _logger?.Error(ex, "Failed to embed album cover"); } file.Save(); return true; } catch (TagLib.CorruptFileException ex) { _logger?.Error(ex, $"File is corrupted or has incorrect extension: {TrackPath}"); return false; } catch (TagLib.UnsupportedFormatException ex) { _logger?.Error(ex, $"File format does not support metadata embedding: {TrackPath}"); return false; } catch (Exception ex) { _logger?.Error(ex, $"Failed to embed metadata in track: {TrackPath}"); return false; } } /// /// Checks if the specified audio format is supported for encoding by FFmpeg. /// /// The audio format to check /// True if the format can be used as a conversion target, false otherwise public static bool IsTargetFormatSupportedForEncoding(AudioFormat format) => BaseConversionParameters.ContainsKey(format); ///// ///// Checks if a given audio format supports embedded metadata tags. ///// ///// The audio format to check ///// True if the format supports metadata tagging, false otherwise public static bool SupportsMetadataEmbedding(AudioFormat format) => format switch { // Formats that DO NOT support metadata embedding AudioFormat.AC3 or AudioFormat.EAC3 or AudioFormat.MIDI => false, // Formats that DO support metadata embedding AudioFormat.AAC or AudioFormat.MP3 or AudioFormat.Opus or AudioFormat.Vorbis or AudioFormat.FLAC or AudioFormat.WAV or AudioFormat.MP4 or AudioFormat.AIFF or AudioFormat.OGG or AudioFormat.WMA or AudioFormat.ALAC or AudioFormat.APE => true, // Unknown formats - assume they might support it _ => true }; /// /// Gets the actual audio codec from a file using FFmpeg and returns the corresponding AudioFormat. /// /// Path to the audio file /// AudioFormat enum value or AudioFormat.Unknown if codec is not supported or detection fails public static async Task GetSupportedCodecAsync(string filePath) { try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(filePath); IAudioStream? audioStream = mediaInfo.AudioStreams.FirstOrDefault(); if (audioStream == null) { NzbDroneLogger.GetLogger(typeof(AudioMetadataHandler)).Debug("No audio stream found in file: {0}", filePath); return AudioFormat.Unknown; } string codec = audioStream.Codec.ToLower(); AudioFormat format = AudioFormatHelper.GetAudioFormatFromCodec(codec); NzbDroneLogger.GetLogger(typeof(AudioMetadataHandler)).Trace("Detected codec '{0}' as format '{1}' for file: {2}", codec, format, filePath); return format; } catch (Exception ex) { NzbDroneLogger.GetLogger(typeof(AudioMetadataHandler)).Error(ex, "Failed to detect codec for file: {0}", filePath); return AudioFormat.Unknown; } } public static bool CheckFFmpegInstalled() { if (_isFFmpegInstalled.HasValue) return _isFFmpegInstalled.Value; bool isInstalled = false; if (!string.IsNullOrEmpty(FFmpeg.ExecutablesPath) && Directory.Exists(FFmpeg.ExecutablesPath)) { string[] ffmpegPatterns = ["ffmpeg", "ffmpeg.exe", "ffmpeg.bin"]; string[] files = Directory.GetFiles(FFmpeg.ExecutablesPath); if (files.Any(file => ffmpegPatterns.Contains(Path.GetFileName(file), StringComparer.OrdinalIgnoreCase) && IsExecutable(file))) { isInstalled = true; } } if (!isInstalled) { string? ffmpegEnv = Environment.GetEnvironmentVariable("FFMPEG"); if (!string.IsNullOrEmpty(ffmpegEnv)) { string dir = File.Exists(ffmpegEnv) ? Path.GetDirectoryName(ffmpegEnv)! : ffmpegEnv; if (Directory.Exists(dir)) { string[] ffmpegPatterns = ["ffmpeg", "ffmpeg.exe", "ffmpeg.bin"]; if (Directory.GetFiles(dir).Any(file => ffmpegPatterns.Contains(Path.GetFileName(file), StringComparer.OrdinalIgnoreCase) && IsExecutable(file))) { FFmpeg.SetExecutablesPath(dir); isInstalled = true; } } } } if (!isInstalled) { foreach (string path in Environment.GetEnvironmentVariable("PATH")?.Split(Path.PathSeparator) ?? []) { if (Directory.Exists(path)) { string[] ffmpegPatterns = ["ffmpeg", "ffmpeg.exe", "ffmpeg.bin"]; string[] files = Directory.GetFiles(path); if (files.Any(file => ffmpegPatterns.Contains(Path.GetFileName(file), StringComparer.OrdinalIgnoreCase) && IsExecutable(file))) { FFmpeg.SetExecutablesPath(path); isInstalled = true; break; } } } } if (!isInstalled) NzbDroneLogger.GetLogger(typeof(AudioMetadataHandler)).Trace("FFmpeg not found in configured path or system PATH"); _isFFmpegInstalled = isInstalled; return isInstalled; } private static bool IsExecutable(string filePath) { try { using FileStream stream = File.OpenRead(filePath); byte[] magicNumber = new byte[4]; stream.Read(magicNumber, 0, 4); // Windows PE if (magicNumber[0] == 0x4D && magicNumber[1] == 0x5A) return true; // Linux ELF if (magicNumber[0] == 0x7F && magicNumber[1] == 0x45 && magicNumber[2] == 0x4C && magicNumber[3] == 0x46) return true; // macOS Mach-O (32-bit: 0xFEEDFACE, 64-bit: 0xFEEDFACF) if (magicNumber[0] == 0xFE && magicNumber[1] == 0xED && magicNumber[2] == 0xFA && (magicNumber[3] == 0xCE || magicNumber[3] == 0xCF)) return true; // Universal Binary (fat_header) if (magicNumber[0] == 0xCA && magicNumber[1] == 0xFE && magicNumber[2] == 0xBA && magicNumber[3] == 0xBE) return true; } catch { } return false; } public static void ResetFFmpegInstallationCheck() => _isFFmpegInstalled = null; public static Task InstallFFmpeg(string path) { NzbDroneLogger.GetLogger(typeof(AudioMetadataHandler)).Trace($"Installing FFmpeg to: {path}"); ResetFFmpegInstallationCheck(); FFmpeg.SetExecutablesPath(path); return CheckFFmpegInstalled() ? Task.CompletedTask : FFmpegDownloader.GetLatestVersion(FFmpegVersion.Official, path); } } } ================================================ FILE: Tubifarry/Core/Model/FileCache.cs ================================================ using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; using System.Text.Json; namespace Tubifarry.Core.Model { public class FileCache { private readonly string _cacheDirectory; private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true }; public FileCache(string cacheDirectory) { if (!Directory.Exists(cacheDirectory)) Directory.CreateDirectory(cacheDirectory); _cacheDirectory = cacheDirectory; CleanupOldCacheFiles(); } /// /// Deletes any JSON files stored directly in the root cache directory, /// as these are from the old file naming scheme. /// private void CleanupOldCacheFiles() { try { foreach (string file in Directory.GetFiles(_cacheDirectory, "*.json", SearchOption.TopDirectoryOnly)) File.Delete(file); } catch { } } /// /// Retrieves cached data for the given key if available and not expired. /// public async Task GetAsync(string cacheKey) { string cacheFilePath = GetCacheFilePath(cacheKey); if (!File.Exists(cacheFilePath)) return default; string json = await File.ReadAllTextAsync(cacheFilePath); CachedData? cachedData = JsonSerializer.Deserialize>(json); if (cachedData == null || DateTime.UtcNow - cachedData.CreatedAt > cachedData.ExpirationDuration) { try { File.Delete(cacheFilePath); } catch { } return default; } return cachedData.Data; } /// /// Caches the provided data with the specified expiration duration. /// public async Task SetAsync(string cacheKey, T data, TimeSpan expirationDuration) { string cacheFilePath = GetCacheFilePath(cacheKey); string directory = Path.GetDirectoryName(cacheFilePath)!; if (!Directory.Exists(directory)) Directory.CreateDirectory(directory); CachedData cachedData = new() { Data = data, CreatedAt = DateTime.UtcNow, ExpirationDuration = expirationDuration }; string json = JsonSerializer.Serialize(cachedData, _jsonOptions); await File.WriteAllTextAsync(cacheFilePath, json); } /// /// Checks whether a valid cache file exists for the given key. /// public bool IsCacheValid(string cacheKey, TimeSpan expirationDuration) { string cacheFilePath = GetCacheFilePath(cacheKey); if (!File.Exists(cacheFilePath)) return false; string json = File.ReadAllText(cacheFilePath); CachedData? cachedData = JsonSerializer.Deserialize>(json); return cachedData != null && DateTime.UtcNow - cachedData.CreatedAt <= expirationDuration; } /// /// Computes the file path for a given cache key using a SHA256 hash and sharding. /// private string GetCacheFilePath(string cacheKey) { byte[] hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(cacheKey)); string hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant(); string subdirectory = hashString[..2]; string fileName = $"{hashString}.json"; return Path.Combine(_cacheDirectory, subdirectory, fileName); } /// /// Ensures that the cache directory is writable and path length is valid. /// public void CheckDirectory() { try { if (!Directory.Exists(_cacheDirectory)) Directory.CreateDirectory(_cacheDirectory); string testFile = Path.Combine(_cacheDirectory, "test.tmp"); File.WriteAllText(testFile, "test"); File.Delete(testFile); int maxPath = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 255 : 4040; int maxCachePathLength = _cacheDirectory.Length + 40; if (maxCachePathLength >= maxPath) throw new PathTooLongException($"Cache path exceeds OS limits ({maxCachePathLength} characters). Use a shorter base directory."); } catch (Exception ex) { throw new InvalidOperationException($"Cache directory validation failed: {ex.Message}", ex); } } } public class CachedData { public T? Data { get; set; } public DateTime CreatedAt { get; set; } public TimeSpan ExpirationDuration { get; set; } } } ================================================ FILE: Tubifarry/Core/Model/PlaylistItem.cs ================================================ namespace Tubifarry.Core.Model; public record PlaylistItem( string ArtistMusicBrainzId, string? AlbumMusicBrainzId, string ArtistName, string? AlbumTitle, string? TrackTitle = null, string? ForeignRecordingId = null); public record PlaylistSnapshot( string ListName, List Items, DateTime FetchedAt); public interface IPlaylistTrackSource { List FetchTrackLevelItems(); } ================================================ FILE: Tubifarry/Core/Model/TrustedSessionException.cs ================================================ namespace Tubifarry.Core.Model { /// /// Exceptions specific to the YouTube trusted session authentication process /// public class TrustedSessionException : Exception { public TrustedSessionException(string message) : base(message) { } public TrustedSessionException(string message, Exception innerException) : base(message, innerException) { } public TrustedSessionException() { } } } ================================================ FILE: Tubifarry/Core/Records/Lyric.cs ================================================ using DownloadAssistant.Base; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NzbDrone.Core.Parser.Model; using System.Text.RegularExpressions; namespace Tubifarry.Core.Records { public record Lyric(string? PlainLyrics, List? SyncedLyrics) { public static async Task FetchLyricsFromLRCLIBAsync(string instance, ReleaseInfo releaseInfo, string trackName, int duration = 0, CancellationToken token = default) { string requestUri = $"{instance}/api/get?artist_name={Uri.EscapeDataString(releaseInfo.Artist)}&track_name={Uri.EscapeDataString(trackName)}&album_name={Uri.EscapeDataString(releaseInfo.Album)}{(duration != 0 ? $"&duration={duration}" : "")}"; HttpResponseMessage response = await HttpGet.HttpClient.GetAsync(requestUri, token); if (!response.IsSuccessStatusCode) return null; JObject json = JObject.Parse(await response.Content.ReadAsStringAsync(token)); return new Lyric(json["plainLyrics"]?.ToString() ?? string.Empty, SyncLine.ParseSyncedLyrics(json["syncedLyrics"]?.ToString() ?? string.Empty)); } } public partial record class SyncLine { [JsonProperty("lrc_timestamp")] public string? LrcTimestamp { get; init; } [JsonProperty("milliseconds")] public string? Milliseconds { get; init; } [JsonProperty("duration")] public string? Duration { get; init; } [JsonProperty("line")] public string? Line { get; init; } public static List ParseSyncedLyrics(string syncedLyrics) { List lyric = []; string[] array = syncedLyrics.Split(new char[1] { '\n' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < array.Length; i++) { Match match = TagRegex().Match(array[i]); if (match.Success) { string value = match.Groups[1].Value; string line = match.Groups[2].Value.Trim(); double totalMilliseconds = TimeSpan.ParseExact(value, "mm\\:ss\\.ff", null).TotalMilliseconds; lyric.Add(new SyncLine { LrcTimestamp = "[" + value + "]", Line = line, Milliseconds = totalMilliseconds.ToString() }); } } return lyric; } [GeneratedRegex("\\[(\\d{2}:\\d{2}\\.\\d{2})\\](.*)")] private static partial Regex TagRegex(); } } ================================================ FILE: Tubifarry/Core/Records/MappingAgent.cs ================================================ namespace Tubifarry.Core.Records { public record MappingAgent { public string UserAgent { get; set; } = Tubifarry.UserAgent; public static T? MapAgent(T? mappingAgent, string userAgent) where T : MappingAgent { if (mappingAgent != null) mappingAgent.UserAgent = userAgent; return mappingAgent; } public static IEnumerable? MapAgent(IEnumerable? mappingAgent, string userAgent) where T : MappingAgent { if (mappingAgent != null) { foreach (T mapping in mappingAgent) mapping.UserAgent = userAgent; } return mappingAgent; } public static List? MapAgent(List? mappingAgent, string userAgent) where T : MappingAgent { if (mappingAgent != null) { foreach (T mapping in mappingAgent) mapping.UserAgent = userAgent; } return mappingAgent; } } } ================================================ FILE: Tubifarry/Core/Records/MusicBrainzData.cs ================================================ using System.Xml.Linq; namespace Tubifarry.Core.Records { public record MusicBrainzSearchItem(string? Title, string? AlbumId, string? Artist, string? ArtistId, DateTime ReleaseDate) { public static MusicBrainzSearchItem FromXml(XElement release, XNamespace ns) { XElement? artistCredit = release.Element(ns + "artist-credit")?.Element(ns + "name-credit")?.Element(ns + "artist"); XElement? releaseGroup = release.Element(ns + "release-group"); return new MusicBrainzSearchItem(release.Element(ns + "title")?.Value, releaseGroup?.Attribute("id")?.Value, artistCredit?.Element(ns + "name")?.Value, artistCredit?.Attribute("id")?.Value, DateTime.TryParse(release.Element(ns + "date")?.Value, out DateTime date) ? date : DateTime.MinValue); } } public record MusicBrainzAlbumItem(string? AlbumId, string? Title, string? Type, string? PrimaryType, List SecondaryTypes, string? Artist, string? ArtistId, DateTime ReleaseDate) { public static MusicBrainzAlbumItem? FromXml(XElement releaseGroup, XNamespace ns) { if (releaseGroup == null) return null; List secondaryTypes = new(); XElement? secondaryTypeList = releaseGroup.Element(ns + "secondary-type-list"); if (secondaryTypeList != null) { secondaryTypes = secondaryTypeList.Elements(ns + "secondary-type") .Select(e => e.Value) .Where(v => !string.IsNullOrWhiteSpace(v)) .ToList(); } return new MusicBrainzAlbumItem( releaseGroup.Attribute("id")?.Value, releaseGroup.Element(ns + "title")?.Value, releaseGroup.Attribute("type")?.Value, releaseGroup.Element(ns + "primary-type")?.Value, secondaryTypes, releaseGroup.Element(ns + "artist-credit")?.Element(ns + "name-credit")?.Element(ns + "artist")?.Element(ns + "name")?.Value, releaseGroup.Element(ns + "artist-credit")?.Element(ns + "name-credit")?.Element(ns + "artist")?.Attribute("id")?.Value, DateTime.TryParse(releaseGroup.Element(ns + "first-release-date")?.Value, out DateTime date) ? date : DateTime.MinValue); } } } ================================================ FILE: Tubifarry/Core/Records/YouTubeSession.cs ================================================ using System.Net; namespace Tubifarry.Core.Records { /// /// Represents session token data for transportation and caching /// public record SessionTokens( string PoToken, string VisitorData, DateTime ExpiryUtc, string Source = "Unknown") { /// /// Checks if the tokens are still valid (not expired) /// public bool IsValid => !IsEmpty && DateTime.UtcNow < ExpiryUtc; /// /// Checks if the tokens are empty /// public bool IsEmpty => string.IsNullOrEmpty(PoToken) || string.IsNullOrEmpty(VisitorData); /// /// Gets the remaining time until expiry /// public TimeSpan TimeUntilExpiry => ExpiryUtc - DateTime.UtcNow; } /// /// Represents client session configuration and state /// public record ClientSessionInfo( SessionTokens? Tokens, Cookie[]? Cookies, string GeographicalLocation = "US") { /// /// Checks if the session has valid authentication data /// public bool HasValidTokens => Tokens?.IsValid == true; /// /// Checks if the session has cookies /// public bool HasCookies => Cookies?.Length > 0; /// /// Gets a summary of the authentication methods available /// public string AuthenticationSummary => $"Tokens: {(HasValidTokens ? "Valid" : "Invalid/Missing")}, " + $"Cookies: {(HasCookies ? $"{Cookies!.Length} cookies" : "None")}"; /// /// Compares this session info with another to detect if authentication has changed /// public bool IsEquivalentTo(ClientSessionInfo? other) { if (other == null) return false; return Tokens?.PoToken == other.Tokens?.PoToken && Tokens?.VisitorData == other.Tokens?.VisitorData && GeographicalLocation == other.GeographicalLocation && CookiesAreEquivalent(Cookies, other.Cookies); } private static bool CookiesAreEquivalent(Cookie[]? cookies1, Cookie[]? cookies2) { if (cookies1 == null && cookies2 == null) return true; if (cookies1 == null || cookies2 == null) return false; if (cookies1.Length != cookies2.Length) return false; return cookies1.Zip(cookies2).All(pair => pair.First.Name == pair.Second.Name && pair.First.Value == pair.Second.Value); } } } ================================================ FILE: Tubifarry/Core/Replacements/ExtendedHttpIndexerBase.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using System.Net; using Sentry; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; namespace Tubifarry.Core.Replacements { /// /// Enhanced generic base class for HTTP indexers with advanced search and result handling /// public abstract class ExtendedHttpIndexerBase : IndexerBase where TSettings : IIndexerSettings, new() where TIndexerPageableRequest : IndexerPageableRequest { protected const int MaxNumResultsPerQuery = 1000; protected readonly IHttpClient _httpClient; protected readonly ISentryHelper _sentry; protected new readonly Logger _logger = null!; public override bool SupportsRss { get; } public override bool SupportsSearch { get; } public abstract int PageSize { get; } public abstract TimeSpan RateLimit { get; } public abstract IIndexerRequestGenerator GetExtendedRequestGenerator(); public abstract IParseIndexerResponse GetParser(); protected ExtendedHttpIndexerBase( IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, ISentryHelper sentry, Logger logger) : base(indexerStatusService, configService, parsingService, logger) { _httpClient = httpClient; _sentry = sentry; _logger = logger; } // Concrete implementation of inherited abstract methods public override async Task> FetchRecent() { if (!SupportsRss) return Array.Empty(); return await FetchReleases(g => g.GetRecentRequests(), true); } public override async Task> Fetch(AlbumSearchCriteria searchCriteria) { if (!SupportsSearch) return Array.Empty(); return await FetchReleases(g => g.GetSearchRequests(searchCriteria)); } public override async Task> Fetch(ArtistSearchCriteria searchCriteria) { if (!SupportsSearch) Array.Empty(); return await FetchReleases(g => g.GetSearchRequests(searchCriteria)); } public override HttpRequest GetDownloadRequest(string link) => new(link); protected virtual async Task> FetchReleases( Func, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false) { var span = _sentry.StartSpan("indexer.fetch"); _sentry.SetSpanTag(span, "indexer.type", GetType().Name); _sentry.SetSpanTag(span, "indexer.is_recent", isRecent.ToString()); List releases = []; string url = string.Empty; TimeSpan minimumBackoff = TimeSpan.FromHours(1); try { IIndexerRequestGenerator generator = GetExtendedRequestGenerator(); IParseIndexerResponse parser = GetParser(); IndexerPageableRequestChain pageableRequestChain = pageableRequestChainSelector(generator); bool fullyUpdated = false; ReleaseInfo? lastReleaseInfo = null; if (isRecent) { lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id); } for (int i = 0; i < pageableRequestChain.Tiers; i++) { IEnumerable pageableRequests = pageableRequestChain.GetTier(i); List tierReleases = []; foreach (TIndexerPageableRequest pageableRequest in pageableRequests) { List pagedReleases = []; foreach (IndexerRequest? request in pageableRequest) { url = request.Url.FullUri; IList page = await FetchPage(request, parser); pagedReleases.AddRange(page); if (ShouldStopFetchingPages(isRecent, page, lastReleaseInfo, pagedReleases, ref fullyUpdated)) break; if (!IsFullPage(page)) break; } tierReleases.AddRange(pagedReleases.Where(IsValidRelease)); } releases.AddRange(tierReleases); if (pageableRequestChain.AreTierResultsUsable(i, tierReleases.Count)) { _logger.Debug($"Tier {i + 1} found {tierReleases.Count} usable results out of total {releases.Count} results. Stopping search."); break; } else if (tierReleases.Count != 0) { _logger.Debug($"Tier {i + 1} found {tierReleases.Count} results out of total {releases.Count}, but doesn't meet usability criteria. Trying next tier."); } else { _logger.Debug($"Tier {i + 1} found no results. Total results so far: {releases.Count}. Trying next tier."); } } if (isRecent && !releases.Empty()) UpdateRssSyncStatus(releases, lastReleaseInfo, fullyUpdated); _indexerStatusService.RecordSuccess(Definition.Id); _sentry.SetSpanData(span, "result.count", releases.Count); _sentry.FinishSpan(span, SpanStatus.Ok); } catch (Exception ex) { HandleException(ex, url, minimumBackoff); _sentry.FinishSpan(span, ex); } return CleanupReleases(releases, isRecent); } protected virtual bool ShouldStopFetchingPages(bool isRecent, IList page, ReleaseInfo? lastReleaseInfo, List pagedReleases, ref bool fullyUpdated) { if (!isRecent) return pagedReleases.Count >= MaxNumResultsPerQuery; if (!page.Any()) return false; if (lastReleaseInfo == null) { fullyUpdated = true; return true; } DateTime oldestReleaseDate = page.Min(v => v.PublishDate); if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) { fullyUpdated = true; return true; } if (pagedReleases.Count >= MaxNumResultsPerQuery && oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) { fullyUpdated = false; return true; } return false; } protected virtual bool IsFullPage(IList page) => PageSize != 0 && page.Count >= PageSize; protected virtual bool IsValidRelease(ReleaseInfo release) { if (release.Title.IsNullOrWhiteSpace()) { _logger.Trace("Invalid Release: '{0}' from indexer: {1}. No title provided.", release.InfoUrl, Definition.Name); return false; } if (release.DownloadUrl.IsNullOrWhiteSpace()) { _logger.Trace("Invalid Release: '{0}' from indexer: {1}. No Download URL provided.", release.Title, Definition.Name); return false; } return true; } private void UpdateRssSyncStatus(List releases, ReleaseInfo? lastReleaseInfo, bool fullyUpdated) { List ordered = [.. releases.OrderByDescending(v => v.PublishDate)]; if (!fullyUpdated && lastReleaseInfo != null) { DateTime gapStart = lastReleaseInfo.PublishDate; DateTime gapEnd = ordered[^1].PublishDate; _logger.Warn("Indexer {0} rss sync didn't cover the period between {1} and {2} UTC. Search may be required.", Definition.Name, gapStart, gapEnd); } lastReleaseInfo = ordered[0]; _indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo); } private void HandleException(Exception ex, string url, TimeSpan minimumBackoff) { switch (ex) { case WebException webException: HandleWebException(webException, url); break; case TooManyRequestsException tooManyRequestsEx: HandleTooManyRequestsException(tooManyRequestsEx, minimumBackoff); break; case HttpException httpException: HandleHttpException(httpException, url); break; case RequestLimitReachedException requestLimitEx: HandleRequestLimitReachedException(requestLimitEx, minimumBackoff); break; case ApiKeyException: _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("Invalid API Key for {0} {1}", this, url); break; case IndexerException indexerEx: _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn(indexerEx, "{0}", url); break; case TaskCanceledException taskCancelledEx: _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn(taskCancelledEx, "Unable to connect to indexer, possibly due to a timeout. {0}", url); break; default: _indexerStatusService.RecordFailure(Definition.Id); ex.WithData("FeedUrl", url); _logger.Error(ex, "An error occurred while processing feed. {0}", url); break; } } private void HandleWebException(WebException webException, string url) { if (webException.Status is WebExceptionStatus.NameResolutionFailure or WebExceptionStatus.ConnectFailure) _indexerStatusService.RecordConnectionFailure(Definition.Id); else _indexerStatusService.RecordFailure(Definition.Id); if (webException.Message.Contains("502") || webException.Message.Contains("503") || webException.Message.Contains("504") || webException.Message.Contains("timed out")) _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message); else _logger.Warn("{0} {1} {2}", this, url, webException.Message); } private void HandleHttpException(HttpException ex, string url) { _indexerStatusService.RecordFailure(Definition.Id); if (ex.Response.HasHttpServerError) _logger.Warn("Unable to connect to {0} at [{1}]. Indexer's server is unavailable. Try again later. {2}", this, url, ex.Message); else _logger.Warn("{0} {1}", this, ex.Message); } private void HandleTooManyRequestsException(TooManyRequestsException ex, TimeSpan minimumBackoff) { TimeSpan retryTime = ex.RetryAfter != TimeSpan.Zero ? ex.RetryAfter : minimumBackoff; _indexerStatusService.RecordFailure(Definition.Id, retryTime); _logger.Warn("API Request Limit reached for {0}. Disabled for {1}", this, retryTime); } private void HandleRequestLimitReachedException(RequestLimitReachedException ex, TimeSpan minimumBackoff) { TimeSpan retryTime = ex.RetryAfter != TimeSpan.Zero ? ex.RetryAfter : minimumBackoff; _indexerStatusService.RecordFailure(Definition.Id, retryTime); _logger.Warn("API Request Limit reached for {0}. Disabled for {1}", this, retryTime); } protected virtual async Task> FetchPage(IndexerRequest request, IParseIndexerResponse parser) { IndexerResponse response = await FetchIndexerResponse(request); try { return [.. parser.ParseResponse(response)]; } catch (Exception ex) { ex.WithData(response.HttpResponse, 128 * 1024); _logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content); throw; } } protected virtual async Task FetchIndexerResponse(IndexerRequest request) { _logger.Debug("Downloading Feed " + request.HttpRequest.ToString(false)); if (request.HttpRequest.RateLimit < RateLimit) request.HttpRequest.RateLimit = RateLimit; request.HttpRequest.RateLimitKey = Definition.Id.ToString(); HttpResponse response = await _httpClient.ExecuteAsync(request.HttpRequest); return new IndexerResponse(request, response); } protected override async Task Test(List failures) => failures.AddIfNotNull(await TestConnection()); protected virtual async Task TestConnection() { try { IParseIndexerResponse parser = GetParser(); IIndexerRequestGenerator generator = GetExtendedRequestGenerator(); IndexerRequest? firstRequest = generator.GetRecentRequests().GetAllTiers().FirstOrDefault()?.FirstOrDefault(); if (firstRequest == null) { return new ValidationFailure(string.Empty, "No rss feed query available. This may be an issue with the indexer or your indexer category settings."); } IList releases = await FetchPage(firstRequest, parser); if (releases.Empty()) { return new ValidationFailure(string.Empty, "Query successful, but no results in the configured categories were returned from your indexer. This may be an issue with the indexer or your indexer category settings."); } } catch (Exception ex) { _logger.Warn(ex, "Unable to connect to indexer"); return new ValidationFailure(string.Empty, $"Unable to connect to indexer: {ex.Message}"); } return null!; } } } ================================================ FILE: Tubifarry/Core/Replacements/FlexibleHttpDispatcher.cs ================================================ using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Common.Http.Proxy; using System.Net; using Tubifarry.Core.Utilities; namespace Tubifarry.Core.Replacements { public class FlexibleHttpDispatcher : ManagedHttpDispatcher, IHttpDispatcher { public const string UA_PARAM = "x-user-agent"; public FlexibleHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, IUserAgentValidator userAgentValidator, ICreateManagedWebProxy createManagedWebProxy, ICertificateValidationService certificateValidationService, IUserAgentBuilder userAgentBuilder, ICacheManager cacheManager, Logger logger) : base(proxySettingsProvider, createManagedWebProxy, certificateValidationService, userAgentBuilder, cacheManager, logger) { userAgentValidator.AddAllowedPattern(@"^[^/]+/\d+(\.\d+)*$"); foreach (string pattern in blacklisted) userAgentValidator.AddBlacklistPattern(pattern); } private readonly string[] blacklisted = [ // Fuzzy matching for Lidarr (catches variations like Lidar, Lidaar, etc.) ".*[a]r+.*", ".*[e]rr+.*", // Fuzzy matching for Tubifarry (catches variations like Tubifary, Tubiferry, etc.) ".*t[uo]b?[iey]?.*", // Additional fuzzy blocking patterns ".*.[fbvd][ae]r+.*", ".*b[o0]t.*", ".*cr[ae]wl[ae]r.*", ".*pr[o0]xy.*", ".*scr[ae]p[ae]r.*" ]; async Task IHttpDispatcher.GetResponseAsync(HttpRequest request, CookieContainer cookies) { ExtractUserAgentFromUrl(request); return await GetResponseAsync(request, cookies); } private static void ExtractUserAgentFromUrl(HttpRequest request) { if (request.Url.Query.IsNullOrWhiteSpace()) return; string[] parts = request.Url.Query.Split('&'); string? uaPart = parts.FirstOrDefault(p => p.StartsWith($"{UA_PARAM}=")); if (uaPart != null) { string userAgent = Uri.UnescapeDataString(uaPart.Split('=')[1]); request.Headers.Set("User-Agent", userAgent); request.Url = request.Url.SetQuery(string.Join("&", parts.Where(p => p != uaPart))); } } protected override void AddRequestHeaders(HttpRequestMessage webRequest, HttpHeader headers) { string userAgent = headers.GetSingleValue("User-Agent"); HttpHeader filtered = []; foreach (KeyValuePair h in headers.Where(h => h.Key != "User-Agent")) filtered.Add(h.Key, h.Value); base.AddRequestHeaders(webRequest, filtered); if (userAgent != null) { webRequest.Headers.UserAgent.Clear(); webRequest.Headers.UserAgent.ParseAdd(userAgent); } } } } ================================================ FILE: Tubifarry/Core/Replacements/IIndexerRequestGenerator.cs ================================================ using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using Tubifarry.Core.Utilities; namespace Tubifarry.Core.Replacements { /// /// Generic indexer request generator interface that supports different request chain types /// public interface IIndexerRequestGenerator where TIndexerPageableRequest : IndexerPageableRequest { /// /// Gets requests for recent releases /// IndexerPageableRequestChain GetRecentRequests(); /// /// Gets search requests for an album /// IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria); /// /// Gets search requests for an artist /// IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria); } } ================================================ FILE: Tubifarry/Core/Telemetry/ISearchContextBuffer.cs ================================================ namespace Tubifarry.Core.Telemetry { public interface ISearchContextBuffer { void LogSearch(string searchId, string query, string? artist, string? album, string strategy, int resultCount); void LogSearchSettings(string searchId, int trackCountFilter, bool normalizedSearch, bool appendYear, bool handleVolumeVariations, bool useFallbackSearch, bool useTrackFallback, int minimumResults, bool hasTemplates); void LogExpectedTracks(string searchId, List trackNames, int expectedCount); void LogParseResult(string searchId, string folderPath, string regexMatchType, int fuzzyArtistScore, int fuzzyAlbumScore, int fuzzyArtistTokenSort, int fuzzyAlbumTokenSort, int priority, string codec, int bitrate, int bitDepth, int trackCountExpected, int trackCountActual, string username, bool hasFreeSlot, int queueLength, List? directoryFiles, bool isInteractive); void UpdateSearchResultCount(string searchId, int actualResultCount); void LogGrab(string searchId, string downloadId, bool isInteractive); SlskdBufferedContext? GetContext(string downloadId); SlskdBufferedContext? GetAndRemoveContext(string downloadId); void AddBreadcrumb(string downloadId, string message); void RecordImport(string albumKey); bool WasRecentlyImported(string albumKey, out int daysSinceImport); } } ================================================ FILE: Tubifarry/Core/Telemetry/ISentryHelper.cs ================================================ namespace Tubifarry.Core.Telemetry { public interface ISentryHelper { bool IsEnabled { get; } // Span operations ISpan? StartSpan(string operation, string? description = null); void FinishSpan(ISpan? span, SpanStatus status = SpanStatus.Ok); void FinishSpan(ISpan? span, Exception ex); void SetSpanData(ISpan? span, string key, object? value); void SetSpanTag(ISpan? span, string key, string value); // Event operations void AddBreadcrumb(string? message, string? category = null); void CaptureException(Exception ex, string? message = null); void CaptureEvent(string message, string[] fingerprint, Dictionary? tags = null, Dictionary? extras = null, SentryLevel level = SentryLevel.Warning); // Context buffer operations (routed to ISearchContextBuffer) void LogSearch(string searchId, string query, string? artist, string? album, string strategy, int resultCount); void LogSearchSettings(string searchId, int trackCountFilter, bool normalizedSearch, bool appendYear, bool handleVolumeVariations, bool useFallbackSearch, bool useTrackFallback, int minimumResults, bool hasTemplates); void LogExpectedTracks(string searchId, List trackNames, int expectedCount); void LogParseResult(string searchId, string folderPath, string regexMatchType, int fuzzyArtistScore, int fuzzyAlbumScore, int fuzzyArtistTokenSort, int fuzzyAlbumTokenSort, int priority, string codec, int bitrate, int bitDepth, int trackCountExpected, int trackCountActual, string username, bool hasFreeSlot, int queueLength, List? directoryFiles, bool isInteractive); void UpdateSearchResultCount(string searchId, int actualResultCount); void LogGrab(string searchId, string downloadId, bool isInteractive); SlskdBufferedContext? GetContext(string downloadId); SlskdBufferedContext? GetAndRemoveContext(string downloadId); void AddContextBreadcrumb(string downloadId, string message); void RecordImport(string albumKey); bool WasRecentlyImported(string albumKey, out int daysSinceImport); } } ================================================ FILE: Tubifarry/Core/Telemetry/NoopSentryHelper.cs ================================================ namespace Tubifarry.Core.Telemetry { public class NoopSentryHelper : ISentryHelper { public bool IsEnabled => false; public ISpan? StartSpan(string operation, string? description = null) => null; public void FinishSpan(ISpan? span, SpanStatus status = SpanStatus.Ok) { } public void FinishSpan(ISpan? span, Exception ex) { } public void SetSpanData(ISpan? span, string key, object? value) { } public void SetSpanTag(ISpan? span, string key, string value) { } public void AddBreadcrumb(string? message, string? category = null) { } public void CaptureException(Exception ex, string? message = null) { } public void CaptureEvent(string message, string[] fingerprint, Dictionary? tags = null, Dictionary? extras = null, SentryLevel level = SentryLevel.Warning) { } public void LogSearch(string searchId, string query, string? artist, string? album, string strategy, int resultCount) { } public void LogSearchSettings(string searchId, int trackCountFilter, bool normalizedSearch, bool appendYear, bool handleVolumeVariations, bool useFallbackSearch, bool useTrackFallback, int minimumResults, bool hasTemplates) { } public void LogExpectedTracks(string searchId, List trackNames, int expectedCount) { } public void LogParseResult(string searchId, string folderPath, string regexMatchType, int fuzzyArtistScore, int fuzzyAlbumScore, int fuzzyArtistTokenSort, int fuzzyAlbumTokenSort, int priority, string codec, int bitrate, int bitDepth, int trackCountExpected, int trackCountActual, string username, bool hasFreeSlot, int queueLength, List? directoryFiles, bool isInteractive) { } public void UpdateSearchResultCount(string searchId, int actualResultCount) { } public void LogGrab(string searchId, string downloadId, bool isInteractive) { } public SlskdBufferedContext? GetContext(string downloadId) => null; public SlskdBufferedContext? GetAndRemoveContext(string downloadId) => null; public void AddContextBreadcrumb(string downloadId, string message) { } public void RecordImport(string albumKey) { } public bool WasRecentlyImported(string albumKey, out int daysSinceImport) { daysSinceImport = 0; return false; } } } ================================================ FILE: Tubifarry/Core/Telemetry/SearchContextBuffer.cs ================================================ #if !MASTER_BRANCH using NzbDrone.Common.Instrumentation; using System.Collections.Concurrent; namespace Tubifarry.Core.Telemetry { public class SearchContextBuffer : ISearchContextBuffer { private readonly ConcurrentDictionary _contextBySearchId = new(); private readonly ConcurrentDictionary _contextByDownloadId = new(); private readonly ConcurrentDictionary _recentImports = new(); private static readonly TimeSpan ContextExpiry = TimeSpan.FromHours(1); private static readonly TimeSpan ImportTrackingExpiry = TimeSpan.FromDays(8); private DateTime _lastCleanup = DateTime.UtcNow; public void LogSearch(string searchId, string query, string? artist, string? album, string strategy, int resultCount) { CleanupIfNeeded(); SlskdBufferedContext context = new() { SearchId = searchId, SearchQuery = query, Artist = artist, Album = album, Strategy = strategy, TotalResults = resultCount }; context.Breadcrumbs.Add($"Search: '{query}' via {strategy} → {resultCount} results"); _contextBySearchId[searchId] = context; } public void LogSearchSettings(string searchId, int trackCountFilter, bool normalizedSearch, bool appendYear, bool handleVolumeVariations, bool useFallbackSearch, bool useTrackFallback, int minimumResults, bool hasTemplates) { if (_contextBySearchId.TryGetValue(searchId, out SlskdBufferedContext? context)) { context.SettingsTrackCountFilter = trackCountFilter; context.SettingsNormalizedSearch = normalizedSearch; context.SettingsAppendYear = appendYear; context.SettingsHandleVolumeVariations = handleVolumeVariations; context.SettingsUseFallbackSearch = useFallbackSearch; context.SettingsUseTrackFallback = useTrackFallback; context.SettingsMinimumResults = minimumResults; context.SettingsHasTemplates = hasTemplates; } } public void LogExpectedTracks(string searchId, List trackNames, int expectedCount) { if (_contextBySearchId.TryGetValue(searchId, out SlskdBufferedContext? context)) { context.ExpectedTracks = trackNames; context.ExpectedTrackCount = expectedCount; } } public void LogParseResult( string searchId, string folderPath, string regexMatchType, int fuzzyArtistScore, int fuzzyAlbumScore, int fuzzyArtistTokenSort, int fuzzyAlbumTokenSort, int priority, string codec, int bitrate, int bitDepth, int trackCountExpected, int trackCountActual, string username, bool hasFreeSlot, int queueLength, List? directoryFiles, bool isInteractive) { if (!_contextBySearchId.TryGetValue(searchId, out SlskdBufferedContext? context)) { context = new SlskdBufferedContext { SearchId = searchId, CreatedAt = DateTime.UtcNow }; _contextBySearchId[searchId] = context; } context.FolderPath = folderPath; context.RegexMatchType = regexMatchType; context.FuzzyArtistScore = fuzzyArtistScore; context.FuzzyAlbumScore = fuzzyAlbumScore; context.FuzzyArtistTokenSort = fuzzyArtistTokenSort; context.FuzzyAlbumTokenSort = fuzzyAlbumTokenSort; context.Priority = priority; context.Codec = codec; context.Bitrate = bitrate; context.BitDepth = bitDepth; context.TrackCountExpected = trackCountExpected; context.TrackCountActual = trackCountActual; context.Username = username; context.HasFreeSlot = hasFreeSlot; context.QueueLength = queueLength; context.DirectoryFiles = directoryFiles; context.IsInteractive = isInteractive; context.AllCandidates.Add(new ParseCandidate { FolderName = Path.GetFileName(folderPath.TrimEnd('\\', '/')), FullPath = folderPath, RegexMatchType = regexMatchType, FuzzyArtist = fuzzyArtistScore, FuzzyAlbum = fuzzyAlbumScore, Priority = priority, TrackCount = trackCountActual, Codec = codec, Username = username, WasGrabbed = false }); } public void UpdateSearchResultCount(string searchId, int actualResultCount) { if (_contextBySearchId.TryGetValue(searchId, out SlskdBufferedContext? context)) { context.TotalResults = actualResultCount; int idx = context.Breadcrumbs.FindIndex(b => b.StartsWith("Search:")); if (idx >= 0) context.Breadcrumbs[idx] = $"Search: '{context.SearchQuery}' via {context.Strategy} → {actualResultCount} results"; } } public void LogGrab(string searchId, string downloadId, bool isInteractive) { NzbDroneLogger.GetLogger(this).Debug($"[SearchContextBuffer] LogGrab: searchId={searchId} knownSearchIds=[{string.Join(",", _contextBySearchId.Keys)}]"); if (_contextBySearchId.TryRemove(searchId, out SlskdBufferedContext? context)) { context.DownloadId = downloadId; context.IsInteractive = isInteractive; // Mark the grabbed candidate ParseCandidate? grabbed = context.AllCandidates.FirstOrDefault(c => c.FullPath == context.FolderPath); if (grabbed != null) grabbed.WasGrabbed = true; // Calculate selection analysis if (context.AllCandidates.Count > 0) { context.OurTopPriority = context.AllCandidates.Max(c => c.Priority); context.GrabbedPriority = grabbed?.Priority; context.LidarrUsedOurTop = context.OurTopPriority == context.GrabbedPriority; // Add summary breadcrumb ParseCandidate best = context.AllCandidates.OrderByDescending(c => c.Priority).First(); int rank = context.AllCandidates.OrderByDescending(c => c.Priority).ToList().IndexOf(grabbed) + 1; context.Breadcrumbs.Add($"Parsed {context.AllCandidates.Count} candidates (best: priority={best.Priority}, regex={best.RegexMatchType})"); context.Breadcrumbs.Add($"Grabbed: '{grabbed?.FolderName}' (priority={grabbed?.Priority}, rank=#{rank}, {(isInteractive ? "interactive" : "auto")})"); } else { context.Breadcrumbs.Add($"Grab: {(isInteractive ? "interactive" : "auto-selected")}"); } _contextByDownloadId[downloadId] = context; } } public SlskdBufferedContext? GetContext(string downloadId) { _contextByDownloadId.TryGetValue(downloadId, out SlskdBufferedContext? context); return context; } public SlskdBufferedContext? GetAndRemoveContext(string downloadId) { if (_contextByDownloadId.TryRemove(downloadId, out SlskdBufferedContext? context)) return context; return null; } public void AddBreadcrumb(string downloadId, string message) { if (_contextByDownloadId.TryGetValue(downloadId, out SlskdBufferedContext? context)) context.Breadcrumbs.Add(message); } public void RecordImport(string albumKey) { CleanupIfNeeded(); _recentImports[albumKey] = DateTime.UtcNow; } public bool WasRecentlyImported(string albumKey, out int daysSinceImport) { daysSinceImport = 0; if (_recentImports.TryGetValue(albumKey, out DateTime importTime)) { TimeSpan elapsed = DateTime.UtcNow - importTime; if (elapsed.TotalDays <= 7) { daysSinceImport = (int)Math.Ceiling(elapsed.TotalDays); return true; } } return false; } private void CleanupIfNeeded() { if (DateTime.UtcNow - _lastCleanup < TimeSpan.FromMinutes(10)) return; _lastCleanup = DateTime.UtcNow; DateTime now = DateTime.UtcNow; List expiredSearchIds = _contextBySearchId .Where(kvp => now - kvp.Value.CreatedAt > ContextExpiry) .Select(kvp => kvp.Key) .ToList(); foreach (string key in expiredSearchIds) _contextBySearchId.TryRemove(key, out _); List expiredDownloadIds = _contextByDownloadId .Where(kvp => now - kvp.Value.CreatedAt > ContextExpiry) .Select(kvp => kvp.Key) .ToList(); foreach (string key in expiredDownloadIds) _contextByDownloadId.TryRemove(key, out _); List expiredImports = _recentImports .Where(kvp => now - kvp.Value > ImportTrackingExpiry) .Select(kvp => kvp.Key) .ToList(); foreach (string key in expiredImports) _recentImports.TryRemove(key, out _); } } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/SentryEventFilter.cs ================================================ #if !MASTER_BRANCH using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Sockets; using System.Threading.Tasks; using Sentry; namespace Tubifarry.Core.Telemetry { public static class SentryEventFilter { private static readonly HashSet FilteredExceptionTypes = new(StringComparer.Ordinal) { "TaskCanceledException", "OperationCanceledException", "TimeoutException", "ThreadAbortException" }; private static readonly HashSet FilteredMessageParts = new(StringComparer.OrdinalIgnoreCase) { "connection refused", "network is unreachable", "download cancelled", "operation was canceled", "request aborted", "the operation has timed out", "connection reset", "connection closed", "broken pipe" }; private static readonly HashSet KnownTransientHttpErrors = new(StringComparer.OrdinalIgnoreCase) { "503", "502", "504", "429" }; public static SentryEvent? FilterEvent(SentryEvent? sentryEvent, SentryHint hint) { if (sentryEvent == null) return null; var ex = sentryEvent.Exception; if (ex != null) { if (FilteredExceptionTypes.Contains(ex.GetType().Name)) return null; if (!string.IsNullOrEmpty(ex.Message) && FilteredMessageParts.Any(p => ex.Message.Contains(p, StringComparison.OrdinalIgnoreCase))) return null; if (ex is HttpRequestException httpEx && httpEx.StatusCode != null) { var statusCode = ((int)httpEx.StatusCode).ToString(); if (KnownTransientHttpErrors.Contains(statusCode)) return null; } if (ex is SocketException socketEx) { var socketError = socketEx.SocketErrorCode.ToString(); if (FilteredMessageParts.Any(p => socketError.Contains(p, StringComparison.OrdinalIgnoreCase))) return null; } } EnrichFingerprint(sentryEvent, ex); return sentryEvent; } private static void EnrichFingerprint(SentryEvent sentryEvent, Exception? ex) { if (sentryEvent.Fingerprint.Any()) return; var fingerprint = new List(); if (ex != null) { fingerprint.Add(ex.GetType().Name); var (operation, _) = ClassifyException(ex); if (operation != null) fingerprint.Add(operation); } else { fingerprint.Add("no_exception"); } fingerprint.Add(PluginInfo.Branch); sentryEvent.SetFingerprint(fingerprint); } private static (string? Operation, string Component) ClassifyException(Exception ex) => ex switch { HttpRequestException => ("http_request", "network"), SocketException => ("socket", "network"), TimeoutException => ("timeout", "general"), IOException => ("io", "filesystem"), UnauthorizedAccessException => ("unauthorized", "security"), ArgumentException => ("invalid_argument", "validation"), InvalidOperationException => ("invalid_operation", "logic"), TaskCanceledException => ("task_cancelled", "async"), AggregateException agg when agg.InnerExceptions.Count == 1 => ClassifyException(agg.InnerExceptions[0]), _ => (null, "unknown") }; } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/SentryHelper.cs ================================================ #if !MASTER_BRANCH namespace Tubifarry.Core.Telemetry { public class SentryHelper : ISentryHelper { private readonly ISearchContextBuffer _contextBuffer; public SentryHelper(ISearchContextBuffer contextBuffer) => _contextBuffer = contextBuffer; public bool IsEnabled => TubifarrySentry.IsEnabled; public ISpan? StartSpan(string operation, string? description = null) { ISpan? parent = SentrySdk.GetSpan(); if (parent == null) return null; ISpan span = parent.StartChild(operation, description ?? operation); span.SetTag("plugin", "tubifarry"); span.SetTag("branch", PluginInfo.Branch); return span; } public void FinishSpan(ISpan? span, SpanStatus status = SpanStatus.Ok) => span?.Finish(status); public void FinishSpan(ISpan? span, Exception ex) { if (span == null) return; SpanStatus status = ex switch { TimeoutException => SpanStatus.DeadlineExceeded, OperationCanceledException => SpanStatus.Cancelled, UnauthorizedAccessException => SpanStatus.PermissionDenied, ArgumentException => SpanStatus.InvalidArgument, _ => SpanStatus.InternalError }; span.Finish(ex, status); } public void SetSpanData(ISpan? span, string key, object? value) { if (span != null && value != null) span.SetExtra(key, value); } public void SetSpanTag(ISpan? span, string key, string value) => span?.SetTag(key, value); public void AddBreadcrumb(string? message, string? category = null) { if (!string.IsNullOrEmpty(message)) SentrySdk.AddBreadcrumb(message, category); } public void CaptureException(Exception ex, string? message = null) { if (!string.IsNullOrEmpty(message)) SentrySdk.CaptureException(ex, scope => scope.SetExtra("message", message)); else SentrySdk.CaptureException(ex); } public void CaptureEvent(string message, string[] fingerprint, Dictionary? tags = null, Dictionary? extras = null, SentryLevel level = SentryLevel.Warning) { SentrySdk.CaptureEvent(new SentryEvent { Message = new SentryMessage { Formatted = message }, Level = level, Fingerprint = fingerprint }, scope => { if (tags != null) foreach ((string? k, string? v) in tags) scope.SetTag(k, v); if (extras != null) foreach ((string? k, object? v) in extras) scope.SetExtra(k, v); }); } // Context buffer delegation public void LogSearch(string searchId, string query, string? artist, string? album, string strategy, int resultCount) => _contextBuffer.LogSearch(searchId, query, artist, album, strategy, resultCount); public void LogSearchSettings(string searchId, int trackCountFilter, bool normalizedSearch, bool appendYear, bool handleVolumeVariations, bool useFallbackSearch, bool useTrackFallback, int minimumResults, bool hasTemplates) => _contextBuffer.LogSearchSettings(searchId, trackCountFilter, normalizedSearch, appendYear, handleVolumeVariations, useFallbackSearch, useTrackFallback, minimumResults, hasTemplates); public void LogExpectedTracks(string searchId, List trackNames, int expectedCount) => _contextBuffer.LogExpectedTracks(searchId, trackNames, expectedCount); public void LogParseResult(string searchId, string folderPath, string regexMatchType, int fuzzyArtistScore, int fuzzyAlbumScore, int fuzzyArtistTokenSort, int fuzzyAlbumTokenSort, int priority, string codec, int bitrate, int bitDepth, int trackCountExpected, int trackCountActual, string username, bool hasFreeSlot, int queueLength, List? directoryFiles, bool isInteractive) => _contextBuffer.LogParseResult(searchId, folderPath, regexMatchType, fuzzyArtistScore, fuzzyAlbumScore, fuzzyArtistTokenSort, fuzzyAlbumTokenSort, priority, codec, bitrate, bitDepth, trackCountExpected, trackCountActual, username, hasFreeSlot, queueLength, directoryFiles, isInteractive); public void UpdateSearchResultCount(string searchId, int actualResultCount) => _contextBuffer.UpdateSearchResultCount(searchId, actualResultCount); public void LogGrab(string searchId, string downloadId, bool isInteractive) => _contextBuffer.LogGrab(searchId, downloadId, isInteractive); public SlskdBufferedContext? GetContext(string downloadId) => _contextBuffer.GetContext(downloadId); public SlskdBufferedContext? GetAndRemoveContext(string downloadId) => _contextBuffer.GetAndRemoveContext(downloadId); public void AddContextBreadcrumb(string downloadId, string message) => _contextBuffer.AddBreadcrumb(downloadId, message); public void RecordImport(string albumKey) => _contextBuffer.RecordImport(albumKey); public bool WasRecentlyImported(string albumKey, out int daysSinceImport) => _contextBuffer.WasRecentlyImported(albumKey, out daysSinceImport); } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/SlskdBufferedContext.cs ================================================ namespace Tubifarry.Core.Telemetry { public class SlskdBufferedContext { // Search phase public string? SearchId { get; set; } public string? SearchQuery { get; set; } public string? Artist { get; set; } public string? Album { get; set; } public string? Strategy { get; set; } public int TotalResults { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; // Parse phase public List ExpectedTracks { get; set; } = []; public int ExpectedTrackCount { get; set; } public string? FolderPath { get; set; } public string? RegexMatchType { get; set; } public int FuzzyArtistScore { get; set; } public int FuzzyAlbumScore { get; set; } public int FuzzyArtistTokenSort { get; set; } public int FuzzyAlbumTokenSort { get; set; } public int Priority { get; set; } public string? Codec { get; set; } public int Bitrate { get; set; } public int BitDepth { get; set; } public int TrackCountExpected { get; set; } public int TrackCountActual { get; set; } public string? Username { get; set; } public bool HasFreeSlot { get; set; } public int QueueLength { get; set; } public List? DirectoryFiles { get; set; } public bool IsInteractive { get; set; } public List AllCandidates { get; set; } = []; // Grab phase public string? DownloadId { get; set; } // Selection analysis public int? OurTopPriority { get; set; } public int? GrabbedPriority { get; set; } public bool? LidarrUsedOurTop { get; set; } // Settings snapshot public int? SettingsTrackCountFilter { get; set; } public bool? SettingsNormalizedSearch { get; set; } public bool? SettingsAppendYear { get; set; } public bool? SettingsHandleVolumeVariations { get; set; } public bool? SettingsUseFallbackSearch { get; set; } public bool? SettingsUseTrackFallback { get; set; } public int? SettingsMinimumResults { get; set; } public bool? SettingsHasTemplates { get; set; } // Breadcrumbs public List Breadcrumbs { get; set; } = []; } public class ParseCandidate { public string FolderName { get; set; } = ""; public string FullPath { get; set; } = ""; public string RegexMatchType { get; set; } = ""; public int FuzzyArtist { get; set; } public int FuzzyAlbum { get; set; } public int Priority { get; set; } public int TrackCount { get; set; } public string Codec { get; set; } = ""; public string Username { get; set; } = ""; public bool WasGrabbed { get; set; } } } ================================================ FILE: Tubifarry/Core/Telemetry/SlskdSentryEvents.cs ================================================ #if !MASTER_BRANCH namespace Tubifarry.Core.Telemetry { public static class SlskdSentryEvents { public enum ImportFailureReason { MissingTracks, UnmatchedTracks, AlbumMatchNotClose, MixedTrackIssues, Unknown } public enum DownloadFailureReason { Timeout, Cancelled, UserOffline, PermissionDenied, Unknown } public static void EmitImportFailed( ISentryHelper sentry, ImportFailureReason failureReason, SlskdBufferedContext? context, List? statusMessages = null) { if (!sentry.IsEnabled) return; string reasonStr = failureReason switch { ImportFailureReason.MissingTracks => "missing_tracks", ImportFailureReason.UnmatchedTracks => "unmatched_tracks", ImportFailureReason.AlbumMatchNotClose => "album_match_not_close", ImportFailureReason.MixedTrackIssues => "mixed_track_issues", _ => "unknown" }; string[] fingerprint = ["slskd-import-failed", reasonStr]; Dictionary tags = new() { ["slskd.failure_reason"] = reasonStr, ["slskd.regex_match"] = context?.RegexMatchType ?? "unknown", ["slskd.interactive"] = (context?.IsInteractive ?? false).ToString().ToLower(), ["slskd.codec"] = context?.Codec ?? "unknown", ["slskd.has_free_slot"] = (context?.HasFreeSlot ?? false).ToString().ToLower(), ["slskd.fuzzy_artist_weak"] = ((context?.FuzzyArtistScore ?? 100) < 90).ToString().ToLower(), ["slskd.fuzzy_album_weak"] = ((context?.FuzzyAlbumScore ?? 100) < 90).ToString().ToLower(), ["slskd.track_count_mismatch"] = ((context?.TrackCountExpected ?? 0) != (context?.TrackCountActual ?? 0)).ToString().ToLower(), ["slskd.had_regex_match"] = (context?.RegexMatchType != "none").ToString().ToLower(), ["slskd.multiple_candidates"] = ((context?.AllCandidates?.Count ?? 0) > 1).ToString().ToLower(), ["slskd.grabbed_top_candidate"] = (context?.LidarrUsedOurTop ?? false).ToString().ToLower() }; Dictionary extras = new() { ["search_query"] = context?.SearchQuery ?? "", ["folder_path"] = context?.FolderPath ?? "", ["fuzzy_artist_score"] = context?.FuzzyArtistScore ?? 0, ["fuzzy_album_score"] = context?.FuzzyAlbumScore ?? 0, ["fuzzy_artist_token"] = context?.FuzzyArtistTokenSort ?? 0, ["fuzzy_album_token"] = context?.FuzzyAlbumTokenSort ?? 0, ["priority"] = context?.Priority ?? 0, ["track_count_expected"] = context?.TrackCountExpected ?? 0, ["track_count_actual"] = context?.TrackCountActual ?? 0, ["username"] = context?.Username ?? "", ["queue_length"] = context?.QueueLength ?? 0, ["search_strategy"] = context?.Strategy ?? "", ["bitrate"] = context?.Bitrate ?? 0, ["bit_depth"] = context?.BitDepth ?? 0, ["expected_tracks"] = context?.ExpectedTracks ?? new List(), ["expected_track_count"] = context?.ExpectedTrackCount ?? 0, ["our_top_priority"] = context?.OurTopPriority ?? 0, ["grabbed_priority"] = context?.GrabbedPriority ?? 0, ["grabbed_was_top_priority"] = context?.LidarrUsedOurTop ?? false, ["candidate_count"] = context?.AllCandidates?.Count ?? 0, ["settings.track_count_filter"] = context?.SettingsTrackCountFilter ?? -1, ["settings.normalized_search"] = context?.SettingsNormalizedSearch ?? false, ["settings.append_year"] = context?.SettingsAppendYear ?? false, ["settings.volume_variations"] = context?.SettingsHandleVolumeVariations ?? false, ["settings.fallback_search"] = context?.SettingsUseFallbackSearch ?? false, ["settings.track_fallback"] = context?.SettingsUseTrackFallback ?? false, ["settings.minimum_results"] = context?.SettingsMinimumResults ?? 0, ["settings.has_templates"] = context?.SettingsHasTemplates ?? false }; if (statusMessages != null && statusMessages.Count > 0) extras["status_messages"] = statusMessages; if (context?.DirectoryFiles != null && context.DirectoryFiles.Count > 0) extras["directory_listing"] = context.DirectoryFiles; if (context?.AllCandidates != null && context.AllCandidates.Count > 0) { extras["candidates"] = context.AllCandidates .OrderByDescending(c => c.Priority) .Select(c => new { folder = c.FolderName, regex = c.RegexMatchType, fuzzy_artist = c.FuzzyArtist, fuzzy_album = c.FuzzyAlbum, priority = c.Priority, tracks = c.TrackCount, codec = c.Codec, username = c.Username, grabbed = c.WasGrabbed }).ToList(); } if (context?.Breadcrumbs != null) { foreach (string breadcrumb in context.Breadcrumbs) sentry.AddBreadcrumb(breadcrumb, "slskd"); } sentry.AddBreadcrumb($"Import Failed: {reasonStr}", "slskd"); string message = $"slskd import failed: {reasonStr} for '{context?.Artist ?? "unknown"} - {context?.Album ?? "unknown"}'"; sentry.CaptureEvent(message, fingerprint, tags, extras, SentryLevel.Warning); } public static void EmitImportSuccess( ISentryHelper sentry, SlskdBufferedContext? context) { if (!sentry.IsEnabled) return; string[] fingerprint = ["slskd-import-success"]; Dictionary tags = new() { ["slskd.regex_match"] = context?.RegexMatchType ?? "unknown", ["slskd.interactive"] = (context?.IsInteractive ?? false).ToString().ToLower(), ["slskd.codec"] = context?.Codec ?? "unknown", ["slskd.has_free_slot"] = (context?.HasFreeSlot ?? false).ToString().ToLower(), ["slskd.grabbed_top_candidate"] = (context?.LidarrUsedOurTop ?? false).ToString().ToLower(), ["slskd.track_count_filter"] = context?.SettingsTrackCountFilter?.ToString() ?? "unknown" }; Dictionary extras = new() { ["search_query"] = context?.SearchQuery ?? "", ["folder_path"] = context?.FolderPath ?? "", ["fuzzy_artist_score"] = context?.FuzzyArtistScore ?? 0, ["fuzzy_album_score"] = context?.FuzzyAlbumScore ?? 0, ["priority"] = context?.Priority ?? 0, ["track_count_expected"] = context?.TrackCountExpected ?? 0, ["track_count_actual"] = context?.TrackCountActual ?? 0, ["candidate_count"] = context?.AllCandidates?.Count ?? 0, ["grabbed_was_top_priority"] = context?.LidarrUsedOurTop ?? false, ["settings.track_count_filter"] = context?.SettingsTrackCountFilter ?? -1, ["settings.normalized_search"] = context?.SettingsNormalizedSearch ?? false, ["settings.append_year"] = context?.SettingsAppendYear ?? false, ["settings.volume_variations"] = context?.SettingsHandleVolumeVariations ?? false, ["settings.fallback_search"] = context?.SettingsUseFallbackSearch ?? false, ["settings.track_fallback"] = context?.SettingsUseTrackFallback ?? false, ["settings.minimum_results"] = context?.SettingsMinimumResults ?? 0, ["settings.has_templates"] = context?.SettingsHasTemplates ?? false }; if (context?.Breadcrumbs != null) { foreach (string breadcrumb in context.Breadcrumbs) sentry.AddBreadcrumb(breadcrumb, "slskd"); } sentry.AddBreadcrumb("Import Success", "slskd"); string message = $"slskd import success: '{context?.Artist ?? "unknown"} - {context?.Album ?? "unknown"}'"; sentry.CaptureEvent(message, fingerprint, tags, extras, SentryLevel.Info); } public static void EmitUserReplaced( ISentryHelper sentry, int daysUntilReplaced, SlskdBufferedContext? originalContext, string replacementSource, string? replacementArtist = null, string? replacementAlbum = null) { if (!sentry.IsEnabled) return; string[] fingerprint = ["slskd-user-replaced"]; Dictionary tags = new() { ["slskd.days_until_replaced"] = daysUntilReplaced.ToString(), ["slskd.interactive"] = (originalContext?.IsInteractive ?? false).ToString().ToLower(), ["slskd.original_codec"] = originalContext?.Codec ?? "unknown", ["slskd.replacement_source"] = replacementSource }; Dictionary extras = new() { ["original_folder_path"] = originalContext?.FolderPath ?? "", ["original_search_query"] = originalContext?.SearchQuery ?? "", ["original_fuzzy_artist"] = originalContext?.FuzzyArtistScore ?? 0, ["original_fuzzy_album"] = originalContext?.FuzzyAlbumScore ?? 0, ["replacement_artist"] = replacementArtist ?? "", ["replacement_album"] = replacementAlbum ?? "" }; string message = $"slskd user replaced album after {daysUntilReplaced} days"; sentry.CaptureEvent(message, fingerprint, tags, extras, SentryLevel.Info); } public static void EmitDownloadFailed( ISentryHelper sentry, DownloadFailureReason errorType, SlskdBufferedContext? context, string? errorMessage = null, int retryCount = 0) { if (!sentry.IsEnabled) return; string errorStr = errorType switch { DownloadFailureReason.Timeout => "timeout", DownloadFailureReason.Cancelled => "cancelled", DownloadFailureReason.UserOffline => "user_offline", DownloadFailureReason.PermissionDenied => "permission_denied", _ => "unknown" }; string[] fingerprint = ["slskd-download-failed", errorStr]; Dictionary tags = new() { ["slskd.error_type"] = errorStr, ["slskd.retry_count"] = retryCount.ToString(), ["slskd.codec"] = context?.Codec ?? "unknown" }; Dictionary extras = new() { ["folder_path"] = context?.FolderPath ?? "", ["username"] = context?.Username ?? "", ["file_count"] = context?.TrackCountActual ?? 0, ["error_message"] = errorMessage ?? "" }; if (context?.Breadcrumbs != null) { foreach (string breadcrumb in context.Breadcrumbs) sentry.AddBreadcrumb(breadcrumb, "slskd"); } sentry.AddBreadcrumb($"Download failed: {errorStr}", "slskd"); string message = $"slskd download failed: {errorStr}"; sentry.CaptureEvent(message, fingerprint, tags, extras, SentryLevel.Warning); } public static DownloadFailureReason CategorizeDownloadError(string? errorMessage) { if (string.IsNullOrEmpty(errorMessage)) return DownloadFailureReason.Unknown; string lower = errorMessage.ToLowerInvariant(); if (lower.Contains("timeout") || lower.Contains("timed out")) return DownloadFailureReason.Timeout; if (lower.Contains("cancel") || lower.Contains("abort")) return DownloadFailureReason.Cancelled; if (lower.Contains("offline") || lower.Contains("not available") || lower.Contains("disconnected")) return DownloadFailureReason.UserOffline; if (lower.Contains("permission") || lower.Contains("denied") || lower.Contains("access")) return DownloadFailureReason.PermissionDenied; return DownloadFailureReason.Unknown; } } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/SlskdTrackingService.cs ================================================ #if !MASTER_BRANCH using NLog; using NzbDrone.Core.Download; using NzbDrone.Core.Download.TrackedDownloads; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; namespace Tubifarry.Core.Telemetry { public class SlskdTrackingService(ISentryHelper sentry, Logger logger) : IHandle, IHandle, IHandle, IHandle { private readonly ISentryHelper _sentry = sentry; private readonly Logger _logger = logger; public void Handle(AlbumGrabbedEvent message) { if (!_sentry.IsEnabled) return; string? downloadId = message.DownloadId; if (string.IsNullOrEmpty(downloadId)) return; string? infoUrl = message.Album?.Release?.InfoUrl; if (string.IsNullOrEmpty(infoUrl)) return; string? searchId = ExtractSearchIdFromInfoUrl(infoUrl); if (string.IsNullOrEmpty(searchId)) return; bool isInteractive = message.Album?.ReleaseSource == ReleaseSourceType.InteractiveSearch; _sentry.LogGrab(searchId, downloadId, isInteractive); _logger.Trace($"Linked search {searchId} to download {downloadId}"); } public void Handle(AlbumImportIncompleteEvent message) { if (!_sentry.IsEnabled) return; TrackedDownload trackedDownload = message.TrackedDownload; if (trackedDownload.State != TrackedDownloadState.ImportFailed) return; if (!IsSlskdDownload(trackedDownload)) return; string? downloadId = trackedDownload.DownloadItem?.DownloadId; if (string.IsNullOrEmpty(downloadId)) return; SlskdSentryEvents.ImportFailureReason failureReason = DetermineFailureReason(trackedDownload); List statusMessages = ExtractStatusMessages(trackedDownload); SlskdBufferedContext? context = _sentry.GetAndRemoveContext(downloadId); SlskdSentryEvents.EmitImportFailed(_sentry, failureReason, context, statusMessages); _logger.Debug($"Tracked import failure for download {downloadId}: {failureReason}"); } public void Handle(TrackImportedEvent message) { if (!_sentry.IsEnabled) return; string? downloadId = message.DownloadId; if (string.IsNullOrEmpty(downloadId)) return; SlskdBufferedContext? context = _sentry.GetContext(downloadId); if (context == null) return; bool hadReplacement = false; // Check for replacement (old files exist) if (message.OldFiles?.Any() == true) { string albumKey = GetAlbumKey(message); if (_sentry.WasRecentlyImported(albumKey, out int daysSinceImport)) { hadReplacement = true; string replacementSource = DetermineReplacementSource(message); SlskdSentryEvents.EmitUserReplaced( _sentry, daysSinceImport, context, replacementSource, message.TrackInfo?.Artist?.Name, message.TrackInfo?.Album?.Title); _logger.Debug($"Tracked user replacement after {daysSinceImport} days"); } } if (!hadReplacement) { SlskdSentryEvents.EmitImportSuccess(_sentry, context); _logger.Debug($"Tracked import success for download {downloadId}"); } string key = GetAlbumKey(message); _sentry.RecordImport(key); _sentry.GetAndRemoveContext(downloadId); } public void Handle(DownloadFailedEvent message) { if (!_sentry.IsEnabled) return; string? downloadId = message.DownloadId; if (string.IsNullOrEmpty(downloadId)) return; SlskdBufferedContext? context = _sentry.GetAndRemoveContext(downloadId); SlskdSentryEvents.DownloadFailureReason errorType = SlskdSentryEvents.CategorizeDownloadError(message.Message); SlskdSentryEvents.EmitDownloadFailed(_sentry, errorType, context, message.Message); _logger.Debug($"Tracked download failure for {downloadId}: {errorType}"); } private static bool IsSlskdDownload(TrackedDownload trackedDownload) { string? indexer = trackedDownload.Indexer; if (string.IsNullOrEmpty(indexer)) return false; return indexer.Contains("slskd", StringComparison.OrdinalIgnoreCase) || indexer.Contains("soulseek", StringComparison.OrdinalIgnoreCase); } private static SlskdSentryEvents.ImportFailureReason DetermineFailureReason(TrackedDownload trackedDownload) { bool hasMissingTracks = trackedDownload.StatusMessages .Any(sm => sm.Messages.Any(m => m.Contains("Has missing tracks", StringComparison.OrdinalIgnoreCase))); bool hasUnmatchedTracks = trackedDownload.StatusMessages .Any(sm => sm.Messages.Any(m => m.Contains("Has unmatched tracks", StringComparison.OrdinalIgnoreCase))); bool hasAlbumMatchNotClose = trackedDownload.StatusMessages .Any(sm => sm.Messages.Any(m => m.Contains("Album match is not close enough", StringComparison.OrdinalIgnoreCase))); if (hasAlbumMatchNotClose) return SlskdSentryEvents.ImportFailureReason.AlbumMatchNotClose; return (hasMissingTracks, hasUnmatchedTracks) switch { (true, true) => SlskdSentryEvents.ImportFailureReason.MixedTrackIssues, (true, false) => SlskdSentryEvents.ImportFailureReason.MissingTracks, (false, true) => SlskdSentryEvents.ImportFailureReason.UnmatchedTracks, _ => SlskdSentryEvents.ImportFailureReason.Unknown }; } private static List ExtractStatusMessages(TrackedDownload trackedDownload) => [.. trackedDownload.StatusMessages .SelectMany(sm => sm.Messages) .Where(m => !string.IsNullOrEmpty(m))]; private static string? ExtractSearchIdFromInfoUrl(string infoUrl) { int searchesIndex = infoUrl.LastIndexOf("/searches/", StringComparison.OrdinalIgnoreCase); if (searchesIndex >= 0) return infoUrl[(searchesIndex + "/searches/".Length)..]; return null; } private static string GetAlbumKey(TrackImportedEvent message) => $"{message.TrackInfo?.Artist?.Id ?? 0}-{message.TrackInfo?.Album?.Id ?? 0}"; private static string DetermineReplacementSource(TrackImportedEvent message) { string? downloadClient = message.DownloadClientInfo?.Name; if (string.IsNullOrEmpty(downloadClient)) return "other"; if (downloadClient.Contains("slskd", StringComparison.OrdinalIgnoreCase) || downloadClient.Contains("soulseek", StringComparison.OrdinalIgnoreCase)) return "slskd"; if (downloadClient.Contains("youtube", StringComparison.OrdinalIgnoreCase)) return "youtube"; return "other"; } } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/TubifarrySentry.cs ================================================ #if !MASTER_BRANCH using NzbDrone.Common; using NzbDrone.Common.EnvironmentInfo; using System.Diagnostics; using System.Runtime.InteropServices; namespace Tubifarry.Core.Telemetry { public static class TubifarrySentry { private static IDisposable? _sdk; private static bool _initialized; private static bool _disabled; private static readonly object _lock = new(); public static bool IsEnabled => _initialized && _sdk != null && !_disabled; public static void Initialize() { lock (_lock) { if (_initialized) return; _initialized = true; if (!PluginInfo.CI || Debugger.IsAttached) { _disabled = true; return; } if (string.IsNullOrEmpty(PluginInfo.SentryDsn)) { _disabled = true; return; } try { _sdk = SentrySdk.Init(o => { o.Dsn = PluginInfo.SentryDsn; o.Release = $"tubifarry@{PluginInfo.InformationalVersion}"; o.Environment = GetEnvironment(); o.AttachStacktrace = true; o.MaxBreadcrumbs = 100; o.AutoSessionTracking = false; o.IsGlobalModeEnabled = false; o.TracesSampleRate = 0.1; o.ProfilesSampleRate = 0; o.SendDefaultPii = false; o.SetBeforeSend((evt, hint) => SentryEventFilter.FilterEvent(evt, hint)); }); ConfigureDefaultScope(); } catch (Exception) { _sdk = null; _disabled = true; } } } private static string GetEnvironment() { if (RuntimeInfo.IsTesting) return "testing"; if (RuntimeInfo.IsDevelopment || Debugger.IsAttached) return "development"; return PluginInfo.Branch; } private static void ConfigureDefaultScope() { try { SentrySdk.ConfigureScope(scope => { scope.User = new SentryUser { Id = HashUtil.AnonymousToken() }; scope.Contexts.App.Name = "Tubifarry"; scope.Contexts.App.Version = PluginInfo.Version; scope.Contexts.App.Build = PluginInfo.GitCommit; scope.SetTag("branch", PluginInfo.Branch); scope.SetTag("plugin_version", PluginInfo.Version); scope.SetTag("lidarr_version", BuildInfo.Version.ToString()); scope.SetTag("runtime_identifier", RuntimeInformation.RuntimeIdentifier); scope.SetTag("culture", Thread.CurrentThread.CurrentCulture.Name); scope.SetTag("ci_build", PluginInfo.CI.ToString()); }); } catch { } } public static void Shutdown() { lock (_lock) { try { _sdk?.Dispose(); } catch { } _sdk = null; _initialized = false; _disabled = false; } } public static void ConfigureScope(Action configure) { if (!IsEnabled) return; try { SentrySdk.ConfigureScope(configure); } catch { } } } } #endif ================================================ FILE: Tubifarry/Core/Telemetry/TubifarrySentryTarget.cs ================================================ #if !MASTER_BRANCH using NLog; using NLog.Targets; namespace Tubifarry.Core.Telemetry { [Target("TubifarrySentry")] public sealed class TubifarrySentryTarget : TargetWithLayout { private static readonly Dictionary LevelMap = new() { { LogLevel.Debug, SentryLevel.Debug }, { LogLevel.Error, SentryLevel.Error }, { LogLevel.Fatal, SentryLevel.Fatal }, { LogLevel.Info, SentryLevel.Info }, { LogLevel.Trace, SentryLevel.Debug }, { LogLevel.Warn, SentryLevel.Warning } }; private static readonly Dictionary BreadcrumbLevelMap = new() { { LogLevel.Debug, BreadcrumbLevel.Debug }, { LogLevel.Error, BreadcrumbLevel.Error }, { LogLevel.Fatal, BreadcrumbLevel.Fatal }, { LogLevel.Info, BreadcrumbLevel.Info }, { LogLevel.Trace, BreadcrumbLevel.Debug }, { LogLevel.Warn, BreadcrumbLevel.Warning } }; public bool Enabled { get; set; } = true; public LogLevel MinimumEventLevel { get; set; } = LogLevel.Error; public LogLevel MinimumBreadcrumbLevel { get; set; } = LogLevel.Debug; protected override void Write(LogEventInfo logEvent) { if (!Enabled || !TubifarrySentry.IsEnabled) return; if (string.IsNullOrEmpty(logEvent.LoggerName) || !logEvent.LoggerName.StartsWith("Tubifarry", StringComparison.Ordinal)) return; try { if (logEvent.Level >= MinimumBreadcrumbLevel) { BreadcrumbLevel breadcrumbLevel = BreadcrumbLevelMap.GetValueOrDefault(logEvent.Level, BreadcrumbLevel.Info); Dictionary? data = logEvent.Properties? .Where(p => p.Key?.ToString() != "Sentry") .ToDictionary(p => p.Key?.ToString() ?? "", p => p.Value?.ToString() ?? ""); SentrySdk.AddBreadcrumb( logEvent.FormattedMessage, logEvent.LoggerName, level: breadcrumbLevel, data: data?.Count > 0 ? data : null); } if (logEvent.Level >= MinimumEventLevel || logEvent.Exception != null) { CaptureEvent(logEvent); } } catch { } } private void CaptureEvent(LogEventInfo logEvent) { SentryEvent sentryEvent = new SentryEvent(logEvent.Exception) { Level = LevelMap.GetValueOrDefault(logEvent.Level, SentryLevel.Info), Logger = logEvent.LoggerName, Message = logEvent.FormattedMessage }; sentryEvent.SetExtra("logger_name", logEvent.LoggerName); if (logEvent.CallerFilePath != null) sentryEvent.SetExtra("caller_file", logEvent.CallerFilePath); if (logEvent.CallerLineNumber > 0) sentryEvent.SetExtra("caller_line", logEvent.CallerLineNumber); if (logEvent.CallerMemberName != null) sentryEvent.SetExtra("caller_member", logEvent.CallerMemberName); if (logEvent.Properties != null) { foreach (KeyValuePair prop in logEvent.Properties) { string? key = prop.Key?.ToString(); if (key == "Sentry" && prop.Value is string[] fingerprint && fingerprint.Length > 0) { sentryEvent.SetFingerprint(fingerprint); } else if (key != null && !string.IsNullOrEmpty(key)) { sentryEvent.SetExtra(key, prop.Value); } } } SentrySdk.CaptureEvent(sentryEvent); } } } #endif ================================================ FILE: Tubifarry/Core/Utilities/AudioFormat.cs ================================================ using Tubifarry.Download.Clients.YouTube; namespace Tubifarry.Core.Utilities { public enum AudioFormat { Unknown, AAC, MP3, Opus, Vorbis, FLAC, WAV, MP4, AIFF, OGG, MIDI, AMR, WMA, ALAC, APE, AC3, EAC3 } internal static class AudioFormatHelper { private static readonly AudioFormat[] _lossyFormats = [ AudioFormat.AAC, AudioFormat.MP3, AudioFormat.Opus, AudioFormat.Vorbis, AudioFormat.MP4, AudioFormat.AMR, AudioFormat.WMA, AudioFormat.AC3, AudioFormat.EAC3 ]; private static readonly int[] _standardBitrates = [ 0, // VBR (Variable Bitrate) 32, // Low-quality voice 64, // Basic music/voice 96, // FM radio quality 128, // Standard streaming (e.g., Spotify Free) 160, // Mid-tier music 192, // High-quality streaming (e.g., YouTube) 256, // Premium streaming (e.g., Spotify Premium) 320, // MP3/AAC upper limit (near-CD quality) 384, // Dolby Digital/AC-3 (5.1 surround) 448, // Dolby Digital/AC-3 (higher-end) 510 // Opus maximum ]; /// /// Defines bitrate constraints for lossy audio formats: (Default, Minimum, Maximum) /// private static readonly Dictionary FormatBitrates = new() { { AudioFormat.AAC, (256, 64, 320) }, { AudioFormat.MP3, (320, 64, 320) }, { AudioFormat.Opus, (256, 32, 510) }, { AudioFormat.Vorbis, (224, 64, 500) }, { AudioFormat.MP4, (256, 64, 320) }, { AudioFormat.AMR, (12, 5, 12) }, { AudioFormat.WMA, (192, 48, 320) }, { AudioFormat.OGG, (224, 64, 500) }, { AudioFormat.AC3, (448, 192, 640) }, { AudioFormat.EAC3, (768, 192, 6144) } }; /// /// Maps approximate bitrates to Vorbis quality levels (-q:a) /// private static readonly Dictionary VorbisBitrateToQuality = new() { { 64, 0 }, // q0 ~64kbps { 80, 1 }, // q1 ~80kbps { 96, 2 }, // q2 ~96kbps { 112, 3 }, // q3 ~112kbps { 128, 4 }, // q4 ~128kbps { 160, 5 }, // q5 ~160kbps { 192, 6 }, // q6 ~192kbps { 224, 7 }, // q7 ~224kbps { 256, 8 }, // q8 ~256kbps { 320, 9 }, // q9 ~320kbps { 500, 10 } // q10 ~500kbps }; /// /// Returns the correct file extension for a given audio codec. /// public static string GetFileExtensionForCodec(string codec) => codec switch { "aac" => ".m4a", "mp3" => ".mp3", "opus" => ".opus", "flac" => ".flac", "ac3" => ".ac3", "eac3" or "ec3" => ".ec3", "alac" => ".m4a", "vorbis" => ".ogg", "ape" => ".ape", "pcm_s16le" or "pcm_s24le" or "pcm_s32le" => ".wav", _ => ".aac" // Default to AAC if the codec is unknown }; /// /// Determines the audio format from a given codec string. /// public static AudioFormat GetAudioFormatFromCodec(string codec) => codec?.ToLowerInvariant() switch { // Common codecs and extensions "aac" or "m4a" or "mp4" => AudioFormat.AAC, "mp3" => AudioFormat.MP3, "opus" => AudioFormat.Opus, "vorbis" or "ogg" => AudioFormat.Vorbis, "flac" => AudioFormat.FLAC, "wav" or "pcm_s16le" or "pcm_s24le" or "pcm_s32le" or "pcm_f32le" => AudioFormat.WAV, "aiff" or "aif" or "aifc" => AudioFormat.AIFF, "mid" or "midi" => AudioFormat.MIDI, "amr" => AudioFormat.AMR, "wma" => AudioFormat.WMA, "alac" => AudioFormat.ALAC, "ape" => AudioFormat.APE, "ac3" or "ac-3" => AudioFormat.AC3, "eac3" or "eac-3" or "e-ac-3" or "ec-3" or "ec3" => AudioFormat.EAC3, _ => AudioFormat.Unknown // Default for unknown formats }; /// /// Returns the file extension for a given audio format. /// public static string GetFileExtensionForFormat(AudioFormat format) => format switch { AudioFormat.AAC => ".m4a", AudioFormat.MP3 => ".mp3", AudioFormat.Opus => ".opus", AudioFormat.Vorbis => ".ogg", AudioFormat.FLAC => ".flac", AudioFormat.WAV => ".wav", AudioFormat.AIFF => ".aiff", AudioFormat.MIDI => ".midi", AudioFormat.AMR => ".amr", AudioFormat.WMA => ".wma", AudioFormat.MP4 => ".mp4", AudioFormat.OGG => ".ogg", AudioFormat.ALAC => ".m4a", AudioFormat.APE => ".ape", AudioFormat.AC3 => ".ac3", AudioFormat.EAC3 => ".ec3", _ => ".aac" // Default to AAC if the format is unknown }; /// /// Converts a value to the corresponding . /// public static AudioFormat ConvertOptionToAudioFormat(ReEncodeOptions reEncodeOption) => reEncodeOption switch { ReEncodeOptions.AAC => AudioFormat.AAC, ReEncodeOptions.MP3 => AudioFormat.MP3, ReEncodeOptions.Opus => AudioFormat.Opus, ReEncodeOptions.Vorbis => AudioFormat.Vorbis, _ => AudioFormat.Unknown }; /// /// Determines if a given format is lossy. /// public static bool IsLossyFormat(AudioFormat format) => _lossyFormats.Contains(format); /// /// Determines the audio format from a given file extension. /// public static AudioFormat GetAudioCodecFromExtension(string extension) => extension?.ToLowerInvariant().TrimStart('.') switch { // Common file extensions "m4a" or "mp4" or "aac" => AudioFormat.AAC, "mp3" => AudioFormat.MP3, "opus" => AudioFormat.Opus, "ogg" or "vorbis" => AudioFormat.Vorbis, "flac" => AudioFormat.FLAC, "wav" => AudioFormat.WAV, "aiff" or "aif" or "aifc" => AudioFormat.AIFF, "mid" or "midi" => AudioFormat.MIDI, "amr" => AudioFormat.AMR, "wma" => AudioFormat.WMA, "alac" => AudioFormat.ALAC, "ape" => AudioFormat.APE, "ac3" => AudioFormat.AC3, "ec3" or "eac3" => AudioFormat.EAC3, _ => AudioFormat.Unknown }; /// /// Returns the default bitrate for a given audio format. /// public static int GetDefaultBitrate(AudioFormat format) => FormatBitrates.TryGetValue(format, out (int Default, int Min, int Max) rates) ? rates.Default : 256; /// /// Clamps a requested bitrate to the valid range for a given format. /// public static int ClampBitrate(AudioFormat format, int requestedBitrate) => !FormatBitrates.TryGetValue(format, out (int Default, int Min, int Max) rates) ? requestedBitrate : Math.Clamp(requestedBitrate, rates.Min, rates.Max); /// /// Maps a target bitrate to the appropriate Vorbis quality level. /// public static int MapBitrateToVorbisQuality(int targetBitrate) => VorbisBitrateToQuality .OrderBy(kvp => Math.Abs(kvp.Key - targetBitrate)) .First().Value; /// /// Rounds a bitrate to the nearest standard value. /// public static int RoundToStandardBitrate(int bitrateKbps) => _standardBitrates.OrderBy(b => Math.Abs(b - bitrateKbps)).First(); } } ================================================ FILE: Tubifarry/Core/Utilities/CacheService.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Annotations; using System.Collections.Concurrent; using Tubifarry.Core.Model; namespace Tubifarry.Core.Utilities { public class CacheService { private readonly ConcurrentDictionary> _memoryCache = new(); private Lazy? _permanentCache; private readonly Logger _logger; public CacheType CacheType { get; set; } = CacheType.Memory; public TimeSpan CacheDuration { get; set; } = TimeSpan.FromDays(7); private string? _cacheDirectory; public string? CacheDirectory { get => _cacheDirectory; set { if (_cacheDirectory != value) { _cacheDirectory = value; _permanentCache = null; } } } public CacheService() => _logger = NzbDroneLogger.GetLogger(this); private FileCache PermanentCache { get { if (_permanentCache == null) { if (string.IsNullOrEmpty(CacheDirectory)) throw new InvalidOperationException("CacheDirectory must be set for permanent cache."); _permanentCache = new Lazy(() => new FileCache(CacheDirectory!)); } return _permanentCache.Value; } } public async Task GetAsync(string key) { if (CacheType == CacheType.Permanent) return await PermanentCache.GetAsync(key); if (_memoryCache.TryGetValue(key, out CachedData? entry) && !IsExpired(entry)) return entry.Data is TData data ? data : default; return default; } public async Task SetAsync(string key, TData value) { if (CacheType == CacheType.Permanent) { await PermanentCache.SetAsync(key, value, CacheDuration); } else { _memoryCache[key] = new CachedData { Data = value!, CreatedAt = DateTime.UtcNow, ExpirationDuration = CacheDuration, }; } } public async Task FetchAndCacheAsync(string key, Func> fetch) { TData? cached = await GetAsync(key); if (cached != null) { _logger.Trace($"Cache hit: {key}"); return cached; } _logger.Debug($"Cache miss: {key}"); TData freshData = await fetch(); await SetAsync(key, freshData); return freshData; } public async Task UpdateAsync(string key, Func> updateFunc) { TData? current = await GetAsync(key); TData updated = await updateFunc(current); await SetAsync(key, updated); return updated; } private bool IsExpired(CachedData entry) => (DateTime.UtcNow - entry.CreatedAt) > CacheDuration; } /// /// Defines the type of cache used by the Search Sniper feature. /// public enum CacheType { /// /// Cache is stored in memory. This is faster but does not persist after the application restarts. /// [FieldOption(Label = "Memory", Hint = "Cache is stored in memory and cleared on application restart. No directory is required.")] Memory = 0, /// /// Cache is stored permanently on disk. This persists across application restarts but requires a valid directory. /// [FieldOption(Label = "Permanent", Hint = "Cache is stored on disk and persists across application restarts. A valid directory is required.")] Permanent = 1 } } ================================================ FILE: Tubifarry/Core/Utilities/CookieManager.cs ================================================ using System.Net; namespace Tubifarry.Core.Utilities { internal static class CookieManager { internal static Cookie[] ParseCookieFile(string filePath) { if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath)) return []; List cookies = []; try { foreach (string line in File.ReadLines(filePath)) { if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#')) continue; string[] parts = line.Split('\t'); if (parts.Length < 7) continue; string domain = parts[0].Trim(); string path = parts[2].Trim(); string secureFlag = parts[3].Trim(); string httpOnlyFlag = parts[1].Trim(); string expiresString = parts[4].Trim(); string name = parts[5].Trim(); string value = parts[6].Trim(); if (string.IsNullOrEmpty(domain) || string.IsNullOrEmpty(path) || string.IsNullOrEmpty(name)) continue; if (!long.TryParse(expiresString, out long expires)) expires = 0; bool isSecure = secureFlag.Equals("TRUE", StringComparison.OrdinalIgnoreCase); bool isHttpOnly = httpOnlyFlag.Equals("TRUE", StringComparison.OrdinalIgnoreCase); Cookie cookie = new(name, value, path, domain) { Secure = isSecure, HttpOnly = isHttpOnly, Expires = expires > 0 ? DateTimeOffset.FromUnixTimeSeconds(expires).DateTime : DateTime.MinValue }; cookies.Add(cookie); } } catch { } return [.. cookies]; } } } ================================================ FILE: Tubifarry/Core/Utilities/DynamicSchemaInjector.cs ================================================ using Lidarr.Http.ClientSchema; using NzbDrone.Core.ThingiProvider; using System.Reflection; namespace Tubifarry.Core.Utilities { public static class DynamicSchemaInjector { private static readonly FieldInfo _mappingsField = typeof(SchemaBuilder).GetField("_mappings", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("SchemaBuilder._mappings field not found. Lidarr internals may have changed."); public static void InjectDynamic(IEnumerable dynamicMappings, string dynamicPrefix) where TSettings : IProviderConfig { Dictionary dict = (Dictionary)_mappingsField.GetValue(null)!; lock (dict) { if (!dict.TryGetValue(typeof(TSettings), out FieldMapping[]? existing)) { SchemaBuilder.ToSchema(Activator.CreateInstance()); dict.TryGetValue(typeof(TSettings), out existing); } FieldMapping[] staticMappings = existing? .Where(m => !m.Field.Name.StartsWith(dynamicPrefix, StringComparison.OrdinalIgnoreCase)) .ToArray() ?? []; dict[typeof(TSettings)] = [.. staticMappings, .. dynamicMappings]; } } } } ================================================ FILE: Tubifarry/Core/Utilities/DynamicStateSettings.cs ================================================ using Lidarr.Http.ClientSchema; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using System.Text.Json; namespace Tubifarry.Core.Utilities { public abstract class DynamicStateSettings : IProviderConfig { [FieldDefinition(0, Label = "States", Type = FieldType.Textbox, Hidden = HiddenType.Hidden)] public string StateJson { get; set; } = "{}"; protected Dictionary GetAllBoolStates() => JsonSerializer.Deserialize>( string.IsNullOrEmpty(StateJson) ? "{}" : StateJson) ?? []; public bool GetBoolState(string key) => GetAllBoolStates().GetValueOrDefault(key); public void SetBoolState(string key, bool value) { Dictionary states = GetAllBoolStates(); states[key] = value; StateJson = JsonSerializer.Serialize(states); } public abstract NzbDroneValidationResult Validate(); public static FieldMapping[] BuildMappings(IEnumerable fields) where TSettings : DynamicStateSettings { List mappings = [ new() { Field = new Field { Name = "stateJson", Label = "States", Type = "textbox", Hidden = "hidden", Order = 0, }, PropertyType = typeof(string), GetterFunc = m => ((TSettings)m).StateJson, SetterFunc = (m, v) => ((TSettings)m).StateJson = v?.ToString() ?? "{}", } ]; int order = 1; foreach (DynamicFieldDefinition def in fields) { string key = def.Key; mappings.Add(def.Type == "checkbox" ? new FieldMapping { Field = new Field { Name = key, Label = def.Label, Type = "checkbox", HelpText = def.HelpText, Advanced = def.Advanced, Order = order++, }, PropertyType = typeof(bool), GetterFunc = m => ((TSettings)m).GetBoolState(key), SetterFunc = (m, v) => ((TSettings)m).SetBoolState(key, Convert.ToBoolean(v)), } : new FieldMapping { Field = new Field { Name = key, Label = def.Label, Type = def.Type, HelpText = def.HelpText, Advanced = def.Advanced, Order = order++, }, PropertyType = typeof(string), GetterFunc = m => ((TSettings)m).GetAllBoolStates().TryGetValue(key, out bool v) ? v : string.Empty, SetterFunc = (m, v) => ((TSettings)m).SetBoolState(key, Convert.ToBoolean(v)), }); } return [.. mappings]; } } public record DynamicFieldDefinition( string Key, string Label, string Type = "checkbox", string? HelpText = null, bool Advanced = false); } ================================================ FILE: Tubifarry/Core/Utilities/FileInfoParser.cs ================================================ using System.Text.RegularExpressions; namespace Tubifarry.Core.Utilities { /// /// Was used for Slskd post-processing of file information, now no longer used but kept for potential future use. /// public partial class FileInfoParser { public string? Artist { get; private set; } public string? Title { get; private set; } public int TrackNumber { get; private set; } public int DiscNumber { get; private set; } public string? Tag { get; private set; } public FileInfoParser(string filePath) { if (string.IsNullOrWhiteSpace(filePath)) throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); string filename = Path.GetFileNameWithoutExtension(filePath); ParseFilename(filename); } private void ParseFilename(string filename) { // Try first set of patterns (space, underscore, dash separators with standard chars) foreach (Regex pattern in PatternsSet1) { Match match = pattern.Match(filename); if (match.Success) { ExtractMatchResults(match); return; } } // Try second set of patterns (space, dash separators with underscore in chars) foreach (Regex pattern in PatternsSet2) { Match match = pattern.Match(filename); if (match.Success) { ExtractMatchResults(match); return; } } } private void ExtractMatchResults(Match match) { Artist = match.Groups["artist"].Success ? match.Groups["artist"].Value.Trim() : string.Empty; Title = match.Groups["title"].Success ? match.Groups["title"].Value.Trim() : string.Empty; TrackNumber = match.Groups["track"].Success ? int.Parse(match.Groups["track"].Value) : 0; Tag = match.Groups["tag"].Success ? match.Groups["tag"].Value.Trim() : string.Empty; if (TrackNumber > 100) { DiscNumber = TrackNumber / 100; TrackNumber %= 100; } } // Pattern set 1: chars = a-z0-9,().&'' with space/underscore/dash separators private static readonly Regex[] PatternsSet1 = [ TrackArtistTitleTagPattern1(), TrackArtistTagTitlePattern1(), TrackArtistTitlePattern1(), ArtistTagTrackTitlePattern1(), ArtistTrackTitleTagPattern1(), ArtistTrackTitlePattern1(), ArtistTitleTagPattern1(), ArtistTagTitlePattern1(), ArtistTitlePattern1(), TrackTitlePattern1(), TrackTagTitlePattern1(), TrackTitleTagPattern1(), TitleOnlyPattern() ]; // Pattern set 2: chars = a-z0-9,().&'_ with space/dash separators private static readonly Regex[] PatternsSet2 = [ TrackArtistTitleTagPattern2(), TrackArtistTagTitlePattern2(), TrackArtistTitlePattern2(), ArtistTagTrackTitlePattern2(), ArtistTrackTitleTagPattern2(), ArtistTrackTitlePattern2(), ArtistTitleTagPattern2(), ArtistTagTitlePattern2(), ArtistTitlePattern2(), TrackTitlePattern2(), TrackTagTitlePattern2(), TrackTitleTagPattern2(), TitleOnlyPattern() ]; // Generated patterns for set 1 (chars include standard chars, sep = space/underscore/dash) [GeneratedRegex(@"^(?\d+)(?[\s_-]+)(?[a-z0-9,\(\)\.&'']+)\k(?[a-z0-9,\(\)\.&'']+)\k<sep>(?<tag>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTitleTagPattern1(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s_-]+)(?<artist>[a-z0-9,\(\)\.&'']+)\k<sep>(?<tag>[a-z0-9,\(\)\.&'']+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTagTitlePattern1(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s_-]+)(?<artist>[a-z0-9,\(\)\.&'']+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTitlePattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<tag>[a-z0-9,\(\)\.&'']+)\k<sep>(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTagTrackTitlePattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)\k<sep>(?<tag>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTrackTitleTagPattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTrackTitlePattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<title>[a-z0-9,\(\)\.&'']+)\k<sep>(?<tag>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTitleTagPattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<tag>[a-z0-9,\(\)\.&'']+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTagTitlePattern1(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.&'']+)(?<sep>[\s_-]+)(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTitlePattern1(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s_-]+)(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTitlePattern1(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s_-]+)(?<tag>[a-z0-9,\(\)\.&'']+)\k<sep>(?<title>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTagTitlePattern1(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s_-]+)(?<title>[a-z0-9,\(\)\.&'']+)\k<sep>(?<tag>[a-z0-9,\(\)\.&'']+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTitleTagPattern1(); // Generated patterns for set 2 (chars include underscore, sep = space/dash only) [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<artist>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<tag>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTitleTagPattern2(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<artist>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<tag>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTagTitlePattern2(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<artist>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackArtistTitlePattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<tag>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTagTrackTitlePattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<tag>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTrackTitleTagPattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<track>\d+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTrackTitlePattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<title>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<tag>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTitleTagPattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<tag>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTagTitlePattern2(); [GeneratedRegex(@"^(?<artist>[a-z0-9,\(\)\.\&''_]+)(?<sep>[\s-]+)(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex ArtistTitlePattern2(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTitlePattern2(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<tag>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<title>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTagTitlePattern2(); [GeneratedRegex(@"^(?<track>\d+)(?<sep>[\s-]+)(?<title>[a-z0-9,\(\)\.\&''_]+)\k<sep>(?<tag>[a-z0-9,\(\)\.\&''_]+)$", RegexOptions.IgnoreCase)] private static partial Regex TrackTitleTagPattern2(); // Title only pattern (shared between sets) [GeneratedRegex(@"^(?<title>.+)$", RegexOptions.IgnoreCase)] private static partial Regex TitleOnlyPattern(); } } ================================================ FILE: Tubifarry/Core/Utilities/IndexerParserHelper.cs ================================================ using NzbDrone.Core.Parser.Model; using System.Text.Json; using Tubifarry.Core.Model; namespace Tubifarry.Core.Utilities { /// <summary> /// Common helper utilities for indexer parsers to reduce code duplication /// </summary> public static class IndexerParserHelper { /// <summary> /// Standard JSON serialization options used across all indexers /// </summary> public static readonly JsonSerializerOptions StandardJsonOptions = new() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString, Converters = { new BooleanConverter() } }; /// <summary> /// Processes a collection of items and converts them to ReleaseInfo /// Common pattern used across multiple indexers /// </summary> /// <typeparam name="T">Type of items to process</typeparam> /// <param name="items">Collection of items to process</param> /// <param name="createData">Function to create AlbumData from item</param> /// <param name="releases">List to add releases to</param> public static void ProcessItems<T>( IList<T>? items, Func<T, AlbumData> createData, List<ReleaseInfo> releases) { if ((items?.Count ?? 0) <= 0) return; foreach (T item in items!) { AlbumData data = createData(item); data.ParseReleaseDate(); releases.Add(data.ToReleaseInfo()); } } /// <summary> /// Determines audio format from file extension or content type/MIME type /// </summary> /// <param name="fileExtension">File extension (e.g., "flac", "mp3")</param> /// <param name="contentType">MIME type or codec (e.g., "audio/flac")</param> /// <param name="defaultFormat">Default format if detection fails</param> /// <returns>Detected AudioFormat</returns> public static AudioFormat DetermineFormat( string? fileExtension, string? contentType, AudioFormat defaultFormat = AudioFormat.MP3) { if (!string.IsNullOrEmpty(fileExtension)) { AudioFormat format = AudioFormatHelper.GetAudioCodecFromExtension(fileExtension); if (format != AudioFormat.Unknown) return format; } if (!string.IsNullOrEmpty(contentType)) { string codec = contentType.Contains('/') ? contentType.Split('/').Last() : contentType; AudioFormat format = AudioFormatHelper.GetAudioFormatFromCodec(codec); if (format != AudioFormat.Unknown) return format; } return defaultFormat; } /// <summary> /// Gets or estimates the size of an audio file/album /// </summary> /// <param name="actualSize">Actual size if known</param> /// <param name="durationSeconds">Duration in seconds</param> /// <param name="bitrateKbps">Bitrate in kbps</param> /// <param name="trackCount">Number of tracks</param> /// <param name="defaultSizePerTrack">Default size per track in bytes (default 50MB)</param> /// <returns>Size in bytes</returns> public static long EstimateSize( long actualSize, long durationSeconds, int bitrateKbps, int trackCount = 1, long defaultSizePerTrack = 50 * 1024 * 1024) { // If actual size is known, use it if (actualSize > 0) return actualSize; if (durationSeconds > 0 && bitrateKbps > 0) return durationSeconds * bitrateKbps * 1000 / 8; if (trackCount > 0) return trackCount * defaultSizePerTrack; return defaultSizePerTrack; } /// <summary> /// Gets quality information (format, bitrate, bit depth) from song data /// </summary> /// <param name="fileExtension">File extension</param> /// <param name="contentType">Content/MIME type</param> /// <param name="reportedBitrate">Bitrate reported by source</param> /// <param name="defaultBitDepth">Default bit depth (16 for CD quality)</param> /// <returns>Tuple of (AudioFormat, Bitrate, BitDepth)</returns> public static (AudioFormat Format, int Bitrate, int BitDepth) GetQualityInfo( string? fileExtension, string? contentType, int reportedBitrate, int defaultBitDepth = 16) { AudioFormat format = DetermineFormat(fileExtension, contentType); int bitrate = reportedBitrate > 0 ? reportedBitrate : AudioFormatHelper.GetDefaultBitrate(format); int bitDepth = defaultBitDepth; // Infer higher quality from bitrate for lossless formats if (format == AudioFormat.FLAC) { if (bitrate >= 2304) bitDepth = 24; // Hi-Res audio else if (bitrate >= 1411) bitDepth = 16; // CD quality } return (format, bitrate, bitDepth); } } } ================================================ FILE: Tubifarry/Core/Utilities/JsonConverters.cs ================================================ using System.Text.Json; using System.Text.Json.Serialization; namespace Tubifarry.Core.Utilities { /// <summary> /// Custom JSON converter that handles both string and numeric values, converting them to string /// </summary> public class StringConverter : JsonConverter<string> { public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { return reader.TokenType switch { JsonTokenType.String => reader.GetString() ?? string.Empty, JsonTokenType.Number => reader.GetInt64().ToString(), JsonTokenType.True => "true", JsonTokenType.False => "false", JsonTokenType.Null => string.Empty, _ => throw new JsonException($"Cannot convert token type {reader.TokenType} to string") }; } public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => writer.WriteStringValue(value); } /// <summary> /// Custom JSON converter for flexible float handling /// </summary> public class FloatConverter : JsonConverter<float> { public override float Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { return reader.TokenType switch { JsonTokenType.Number => (float)reader.GetDouble(), JsonTokenType.String => float.TryParse(reader.GetString(), out float result) ? result : throw new JsonException($"Cannot convert string '{reader.GetString()}' to float"), _ => throw new JsonException($"Cannot convert token type {reader.TokenType} to float") }; } public override void Write(Utf8JsonWriter writer, float value, JsonSerializerOptions options) => writer.WriteNumberValue(value); } /// <summary> /// Custom JSON converter that handles both boolean and string boolean values /// </summary> public class BooleanConverter : JsonConverter<bool> { public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { return reader.TokenType switch { JsonTokenType.True => true, JsonTokenType.False => false, JsonTokenType.String => bool.TryParse(reader.GetString(), out bool result) && result, JsonTokenType.Number => reader.GetInt32() != 0, _ => throw new JsonException($"Cannot convert token type {reader.TokenType} to boolean") }; } public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) => writer.WriteBooleanValue(value); } /// <summary> /// Converts Unix timestamps (milliseconds) to DateTime /// </summary> internal class UnixTimestampConverter : JsonConverter<DateTime?> { private static readonly DateTime UnixEpoch = new(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); public override DateTime? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.Null) return null; if (reader.TokenType == JsonTokenType.Number && reader.TryGetInt64(out long unixTime)) return UnixEpoch.AddMilliseconds(unixTime); if (reader.TokenType == JsonTokenType.String) { string? dateStr = reader.GetString(); if (long.TryParse(dateStr, out long unixTimeParsed)) return UnixEpoch.AddMilliseconds(unixTimeParsed); } return null; } public override void Write(Utf8JsonWriter writer, DateTime? value, JsonSerializerOptions options) { if (value.HasValue) { long unixTime = (long)(value.Value.ToUniversalTime() - UnixEpoch).TotalMilliseconds; writer.WriteNumberValue(unixTime); } else { writer.WriteNullValue(); } } } } ================================================ FILE: Tubifarry/Core/Utilities/LazyRequestChain.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers; using System.Collections; namespace Tubifarry.Core.Utilities { /// <summary> /// Lazy version of IndexerPageableRequest that defers execution until enumerated /// </summary> public class LazyIndexerPageableRequest(Func<IEnumerable<IndexerRequest>> requestFactory, int minimumResultsThreshold = 0) : IndexerPageableRequest(new LazyEnumerable(requestFactory)) { public int MinimumResultsThreshold { get; } = minimumResultsThreshold; public bool AreResultsUsable(int resultsCount) => MinimumResultsThreshold == 0 ? resultsCount > 0 : resultsCount >= MinimumResultsThreshold; /// <summary> /// Helper class that wraps the lazy factory for the base constructor /// </summary> private class LazyEnumerable(Func<IEnumerable<IndexerRequest>> factory) : IEnumerable<IndexerRequest> { private readonly Func<IEnumerable<IndexerRequest>> _factory = factory; public IEnumerator<IndexerRequest> GetEnumerator() => _factory().GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } /// <summary> /// Generic version of IndexerPageableRequestChain with proper Add implementation /// </summary> public class IndexerPageableRequestChain<TRequest> where TRequest : IndexerPageableRequest { protected List<List<TRequest>> _chains; private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(IndexerPageableRequestChain<TRequest>)); public IndexerPageableRequestChain() => _chains = [[]]; public virtual int Tiers => _chains.Count; public virtual IEnumerable<TRequest> GetAllTiers() => _chains.SelectMany(v => v); public virtual IEnumerable<TRequest> GetTier(int index) => index < _chains.Count ? _chains[index] : Enumerable.Empty<TRequest>(); public virtual void Add(IEnumerable<IndexerRequest> request) { if (request == null) return; if (typeof(TRequest) == typeof(LazyIndexerPageableRequest) && new LazyIndexerPageableRequest(() => request) is TRequest lazyRequest) { _chains[^1].Add(lazyRequest); Logger.Trace($"Added request to current tier. Current tier now has {_chains[^1].Count} requests."); } } public virtual void AddTier(IEnumerable<IndexerRequest> request) { AddTier(); Add(request); } public virtual void AddTier() { if (_chains[^1].Count == 0) return; _chains.Add([]); Logger.Trace($"Added new tier. Total tiers: {_chains.Count}"); } /// <summary> /// Determines if results from the specified tier are usable based on tier-specific criteria /// </summary> public virtual bool AreTierResultsUsable(int tierIndex, int resultsCount) => resultsCount > 0; /// <summary> /// Converts to standard IndexerPageableRequestChain for compatibility /// </summary> public virtual IndexerPageableRequestChain ToStandardChain() { IndexerPageableRequestChain standardChain = new(); if (_chains.Count > 0 && _chains[0].Count > 0) { foreach (TRequest request in _chains[0]) standardChain.Add(request); } for (int i = 1; i < _chains.Count; i++) { if (_chains[i].Count > 0) { standardChain.AddTier(); foreach (TRequest request in _chains[i]) standardChain.Add(request); } } return standardChain; } } /// <summary> /// Lazy request chain that generates tiers on-demand /// </summary> public class LazyIndexerPageableRequestChain(int defaultThreshold = 0) : IndexerPageableRequestChain<LazyIndexerPageableRequest> { private readonly int _defaultThreshold = defaultThreshold; private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(LazyIndexerPageableRequestChain)); /// <summary> /// Add a request factory to the current tier /// </summary> public void AddFactory(Func<IEnumerable<IndexerRequest>> requestFactory, int minimumResultsThreshold = 0) { int threshold = minimumResultsThreshold == 0 ? _defaultThreshold : minimumResultsThreshold; LazyIndexerPageableRequest lazyRequest = new(requestFactory, threshold); _chains[^1].Add(lazyRequest); Logger.Trace($"Added factory to current tier. Current tier now has {_chains[^1].Count} requests."); } /// <summary> /// Add a new tier with a request factory /// </summary> public void AddTierFactory(Func<IEnumerable<IndexerRequest>> requestFactory, int minimumResultsThreshold = 0) { AddTier(); AddFactory(requestFactory, minimumResultsThreshold); } public override bool AreTierResultsUsable(int tierIndex, int resultsCount) { if (tierIndex >= _chains.Count) return false; int maxThreshold = 0; foreach (LazyIndexerPageableRequest request in _chains[tierIndex]) { if (request.MinimumResultsThreshold > maxThreshold) maxThreshold = request.MinimumResultsThreshold; } return maxThreshold == 0 ? resultsCount > 0 : resultsCount >= maxThreshold; } public override void Add(IEnumerable<IndexerRequest> request) { if (request != null) AddFactory(() => request); } public override void AddTier(IEnumerable<IndexerRequest> request) { if (request != null) AddTierFactory(() => request); } } /// <summary> /// Search tier generator that creates request factories instead of materialized requests /// </summary> public static class SearchTierGenerator { /// <summary> /// Creates a conditional tier that only executes if the condition is met /// </summary> public static Func<IEnumerable<IndexerRequest>> CreateConditionalTier(Func<bool> condition, Func<IEnumerable<IndexerRequest>> requestGenerator) => () => condition() ? requestGenerator() : []; /// <summary> /// Creates a simple tier that always executes /// </summary> public static Func<IEnumerable<IndexerRequest>> CreateTier(Func<IEnumerable<IndexerRequest>> requestGenerator) => requestGenerator; } } ================================================ FILE: Tubifarry/Core/Utilities/PermissionTester.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; namespace Tubifarry.Core.Utilities { public static class PermissionTester { public static ValidationFailure? TestReadWritePermissions(string directoryPath, ILogger logger) { string testFilePath = Path.Combine(directoryPath, "test_permissions.tmp"); try { File.WriteAllText(testFilePath, "This is a test file to check write permissions."); logger.Trace("Write permission test succeeded."); } catch (UnauthorizedAccessException ex) { logger.Warn(ex, "Write permission denied for directory"); return new ValidationFailure("DirectoryPath", $"Write permission denied for directory: {ex.Message}"); } try { string content = File.ReadAllText(testFilePath); logger.Trace("Read permission test succeeded."); } catch (UnauthorizedAccessException ex) { logger.Warn(ex, "Read permission denied for directory"); return new ValidationFailure("DirectoryPath", $"Read permission denied for directory: {ex.Message}"); } try { File.Delete(testFilePath); logger.Trace("Delete permission test succeeded."); } catch (UnauthorizedAccessException ex) { logger.Warn(ex, "Delete permission denied for directory"); return new ValidationFailure("DirectoryPath", $"Delete permission denied for directory: {ex.Message}"); } return null; } public static ValidationFailure? TestExistance(string directoryPath, ILogger logger) { if (string.IsNullOrWhiteSpace(directoryPath)) { logger.Warn("Directory path is null or empty."); return new ValidationFailure("DirectoryPath", "Directory path cannot be null or empty."); } if (!Directory.Exists(directoryPath)) { logger.Info("Directory does not exist. Attempting to create it."); Directory.CreateDirectory(directoryPath); } return null; } public static ValidationFailure? TestExecutePermissions(string directoryPath, ILogger logger) { try { string[] files = Directory.GetFiles(directoryPath); logger.Info("Execute permission test succeeded."); } catch (UnauthorizedAccessException ex) { logger.Warn(ex, "Execute permission denied for directory"); return new ValidationFailure("DirectoryPath", $"Execute permission denied for directory: {ex.Message}"); } return null; } public static List<ValidationFailure> TestAllPermissions(string directoryPath, ILogger logger) { List<ValidationFailure> tests = []; try { tests!.AddIfNotNull(TestExistance(directoryPath, logger)); tests!.AddIfNotNull(TestReadWritePermissions(directoryPath, logger)); tests!.AddIfNotNull(TestExecutePermissions(directoryPath, logger)); logger.Trace("All directory permissions tests succeeded."); } catch (IOException ex) { logger.Warn(ex, "IO error while testing directory permissions"); tests.Add(new ValidationFailure("DirectoryPath", $"IO error while testing directory: {ex.Message}")); } catch (Exception ex) { logger.Error(ex, "Unexpected error while testing directory permissions"); tests.Add(new ValidationFailure("DirectoryPath", $"Unexpected error: {ex.Message}")); } return tests; } } } ================================================ FILE: Tubifarry/Core/Utilities/PluginSettings.cs ================================================ using NLog; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using System.Collections.Concurrent; using System.Text; using System.Text.Json; namespace Tubifarry.Core.Utilities { public interface IPluginSettings { T GetValue<T>(string key, T? defaultValue = default); void SetValue<T>(string key, T value); bool HasKey(string key); void RemoveKey(string key); void Save(); void Load(); event EventHandler<SettingChangedEventArgs> SettingChanged; } public class SettingChangedEventArgs(string key, object? oldValue, object? newValue) : EventArgs { public string Key { get; } = key; public object? OldValue { get; } = oldValue; public object? NewValue { get; } = newValue; } public class PluginSettings : IPluginSettings { private readonly string _settingsPath; private readonly ConcurrentDictionary<string, string> _settings; private readonly object _syncLock = new(); private readonly JsonSerializerOptions _jsonOptions; private readonly bool _autoSave; private readonly Logger _logger; public event EventHandler<SettingChangedEventArgs>? SettingChanged; public PluginSettings(IAppFolderInfo appFolderInfo, Logger logger, bool autoSave = true) { _logger = logger; _settingsPath = Path.Combine(appFolderInfo.GetPluginPath(), PluginInfo.Author, PluginInfo.Name, "settings.resx"); _settings = []; _autoSave = autoSave; _jsonOptions = new JsonSerializerOptions { WriteIndented = false, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; Load(); } public T GetValue<T>(string key, T? defaultValue = default) { lock (_syncLock) { return _settings.TryGetValue(key, out string? json) ? TryDeserialize(json, defaultValue)! : defaultValue!; } } private T? TryDeserialize<T>(string json, T? defaultValue) { try { return JsonSerializer.Deserialize<T>(json, _jsonOptions) is T result ? result : defaultValue; } catch { return defaultValue; } } public void SetValue<T>(string key, T value) { lock (_syncLock) { string json = JsonSerializer.Serialize(value, _jsonOptions); T? oldValue = _settings.TryGetValue(key, out string? oldJson) && oldJson != null ? TryDeserialize<T>(oldJson, default) : default; _settings[key] = json; OnSettingChanged(key, oldValue, value); if (_autoSave) SaveInternal(); } } public bool HasKey(string key) { lock (_syncLock) { return _settings.ContainsKey(key); } } public void RemoveKey(string key) { lock (_syncLock) { if (_settings.TryRemove(key, out string? oldJson) && oldJson != null) { object? oldValue = TryDeserialize<object>(oldJson, null); OnSettingChanged(key, oldValue, null); if (_autoSave) { SaveInternal(); } } } } public void Save() { lock (_syncLock) { SaveInternal(); } } private void SaveInternal() { try { string? directory = Path.GetDirectoryName(_settingsPath); if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } string json = JsonSerializer.Serialize(_settings, _jsonOptions); string obfuscated = ObfuscateString(json); File.WriteAllText(_settingsPath, obfuscated); } catch (Exception ex) { _logger.Error(ex, "Error saving plugin settings"); } } public void Load() { lock (_syncLock) { if (!File.Exists(_settingsPath)) { return; } try { string obfuscated = File.ReadAllText(_settingsPath); string json = DeobfuscateString(obfuscated); Dictionary<string, string>? loadedSettings = JsonSerializer.Deserialize<Dictionary<string, string>>(json, _jsonOptions); _settings.Clear(); if (loadedSettings != null) { foreach (KeyValuePair<string, string> pair in loadedSettings) { _settings[pair.Key] = pair.Value; } } } catch (Exception ex) { _logger.Error(ex, "Error loading plugin settings"); _settings.Clear(); } } } public int GetInt(string key, int defaultValue = 0) => GetValue(key, defaultValue); public bool GetBool(string key, bool defaultValue = false) => GetValue(key, defaultValue); public string GetString(string key, string defaultValue = "") => GetValue(key, defaultValue) ?? defaultValue; public double GetDouble(string key, double defaultValue = 0.0) => GetValue(key, defaultValue); public void SetValues<T>(Dictionary<string, T> values) { lock (_syncLock) { values.ToList().ForEach(pair => { T? oldValue = _settings.TryGetValue(pair.Key, out string? oldJson) && oldJson != null ? TryDeserialize<T>(oldJson, default) : default; _settings[pair.Key] = JsonSerializer.Serialize(pair.Value, _jsonOptions); OnSettingChanged(pair.Key, oldValue, pair.Value); }); if (_autoSave) SaveInternal(); } } public void Clear() { lock (_syncLock) { _settings.Clear(); if (_autoSave) SaveInternal(); } } protected virtual void OnSettingChanged(string key, object? oldValue, object? newValue) => SettingChanged?.Invoke(this, new SettingChangedEventArgs(key, oldValue, newValue)); private static string ObfuscateString(string input) => Convert.ToBase64String(Encoding.UTF8.GetBytes(input).Select((b, i) => { byte key = (byte)((i % 256) ^ 0x5F); return (byte)((b ^ key) + 1); }).ToArray()); private static string DeobfuscateString(string input) => Encoding.UTF8.GetString(Convert.FromBase64String(input).Select((b, i) => { byte key = (byte)((i % 256) ^ 0x5F); return (byte)((b - 1) ^ key); }).ToArray()); } } ================================================ FILE: Tubifarry/Core/Utilities/ReleaseFormatter.cs ================================================ using NzbDrone.Core.Music; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using System.Text.RegularExpressions; namespace Tubifarry.Core.Utilities { public partial class ReleaseFormatter(ReleaseInfo releaseInfo, Artist artist, NamingConfig? namingConfig) { private readonly ReleaseInfo _releaseInfo = releaseInfo; private readonly Artist _artist = artist; private readonly NamingConfig? _namingConfig = namingConfig; public string BuildTrackFilename(string? pattern, Track track, Album album) { pattern ??= _namingConfig?.StandardTrackFormat ?? "{track:0} {Track Title}"; Dictionary<string, Func<string>> tokenHandlers = GetTokenHandlers(track, album); string formattedString = ReplaceTokens(pattern, tokenHandlers); return CleanFileName(Path.GetFileName(formattedString)); } public string BuildAlbumFilename(string? pattern, Album album) { pattern ??= "{Album Title}"; Dictionary<string, Func<string>> tokenHandlers = GetTokenHandlers(null, album); string formattedString = ReplaceTokens(pattern, tokenHandlers); return CleanFileName(formattedString); } public string BuildArtistFolderName(string? pattern) { pattern ??= _namingConfig?.ArtistFolderFormat ?? "{Artist Name}"; Dictionary<string, Func<string>> tokenHandlers = GetTokenHandlers(null, null); string formattedString = ReplaceTokens(pattern, tokenHandlers); return CleanFileName(formattedString); } private Dictionary<string, Func<string>> GetTokenHandlers(Track? track, Album? album) => new(StringComparer.OrdinalIgnoreCase) { // Album Tokens (only added if album is provided) { "{Album Title}", () => CleanTitle(album?.Title) }, { "{Album CleanTitle}", () => CleanTitle(album?.Title) }, { "{Album TitleThe}", () => CleanTitle(TitleThe(album?.Title)) }, { "{Album CleanTitleThe}", () => CleanTitle(album?.Title) }, { "{Album Type}", () => CleanTitle(album?.AlbumType) }, { "{Album Genre}", () => CleanTitle(album?.Genres?.FirstOrDefault()) }, { "{Album MbId}", () => album?.ForeignAlbumId ?? string.Empty }, { "{Album Disambiguation}", () => CleanTitle(album?.Disambiguation) }, { "{Release Year}", () => album?.ReleaseDate?.Year.ToString() ?? string.Empty }, // Artist Tokens { "{Artist Name}", () => CleanTitle(_artist?.Name) }, { "{Artist CleanName}", () => CleanTitle(_artist?.Name) }, { "{Artist NameThe}", () => CleanTitle(TitleThe(_artist?.Name))}, { "{Artist CleanNameThe}", () => CleanTitleThe(_artist?.Name) }, { "{Artist Genre}", () => CleanTitle(_artist?.Metadata?.Value?.Genres?.FirstOrDefault()) }, { "{Artist MbId}", () => _artist?.ForeignArtistId ?? string.Empty }, { "{Artist Disambiguation}", () => CleanTitle(_artist?.Metadata?.Value?.Disambiguation) }, { "{Artist NameFirstCharacter}", () => TitleFirstCharacter(_artist?.Name) }, // Track Tokens (only added if track is provided) { "{Track Title}", () => CleanTitle(track?.Title) }, { "{Track CleanTitle}", () => CleanTitle(track?.Title) }, { "{Track ArtistName}", () => CleanTitle(_artist?.Name) }, { "{Track ArtistNameThe}", () => CleanTitle(TitleThe(_artist?.Name)) }, { "{Track ArtistMbId}", () => _artist?.ForeignArtistId ?? string.Empty }, { "{track:0}", () => FormatTrackNumber(track?.TrackNumber, "0") }, { "{track:00}", () => FormatTrackNumber(track?.TrackNumber, "00") }, // Medium Tokens (only added if track is provided) { "{Medium Name}", () => CleanTitle(track?.AlbumRelease?.Value?.Media?.FirstOrDefault(m => m.Number == track.MediumNumber)?.Name) }, { "{medium:0}", () => track?.MediumNumber.ToString("0") ?? string.Empty }, { "{medium:00}", () => track?.MediumNumber.ToString("00") ?? string.Empty }, // Release Info Tokens { "{Original Title}", () => CleanTitle(_releaseInfo?.Title) } }; private static string ReplaceTokens(string pattern, Dictionary<string, Func<string>> tokenHandlers) => ReplaceTokensRegex().Replace(pattern, match => { string token = match.Groups[1].Value; return tokenHandlers.TryGetValue($"{{{token}}}", out Func<string>? handler) ? handler() : string.Empty; }); private string CleanFileName(string fileName) { char[] invalidFileNameChars = Path.GetInvalidFileNameChars(); char[] invalidPathChars = Path.GetInvalidPathChars(); char[] invalidChars = invalidFileNameChars.Union(invalidPathChars).ToArray(); fileName = invalidChars.Aggregate(fileName, (current, invalidChar) => current.Replace(invalidChar.ToString(), string.Empty)); switch (_namingConfig?.ColonReplacementFormat) { case ColonReplacementFormat.Delete: fileName = fileName.Replace(":", string.Empty); break; case ColonReplacementFormat.Dash: fileName = fileName.Replace(":", "-"); break; case ColonReplacementFormat.SpaceDash: fileName = fileName.Replace(":", " -"); break; case ColonReplacementFormat.SpaceDashSpace: fileName = fileName.Replace(":", " - "); break; case ColonReplacementFormat.Smart: fileName = ColonReplaceRegex().Replace(fileName, " - "); break; } return fileName.Trim(); } private static string CleanTitle(string? title) { if (string.IsNullOrEmpty(title)) return string.Empty; return title.Replace("&", "and").Replace("/", " - ").Replace("\\", " - ").Trim(); } private static string TitleThe(string? title) { if (string.IsNullOrEmpty(title)) return string.Empty; return TitleTheRegex().Replace(title, "$2, $1"); } private static string CleanTitleThe(string? title) => CleanTitle(TitleThe(title)); private static string TitleFirstCharacter(string? title) { if (string.IsNullOrEmpty(title)) return "_"; return char.IsLetterOrDigit(title[0]) ? title[..1].ToUpper() : "_"; } private static string FormatTrackNumber(string? trackNumber, string? format) { if (string.IsNullOrEmpty(trackNumber)) return string.Empty; if (int.TryParse(trackNumber, out int trackNumberInt)) return trackNumberInt.ToString(format); return trackNumber; } [GeneratedRegex(@"\{([^}]+)\}")] private static partial Regex ReplaceTokensRegex(); [GeneratedRegex(@"^(The|A|An)\s+(.+)$", RegexOptions.IgnoreCase, "de-DE")] private static partial Regex TitleTheRegex(); [GeneratedRegex(@":\s*")] private static partial Regex ColonReplaceRegex(); } } ================================================ FILE: Tubifarry/Core/Utilities/RepositorySettingsResolver.cs ================================================ using Microsoft.Extensions.DependencyInjection; using NLog; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Core.Utilities { public interface IRepositorySettingsResolver { TSettings Resolve<TRepository, TSettings>(string implementationName) where TRepository : class where TSettings : IProviderConfig; } public class RepositorySettingsResolver(Lazy<IServiceProvider> serviceProvider, Logger logger) : IRepositorySettingsResolver { private readonly Lazy<IServiceProvider> _serviceProvider = serviceProvider; private readonly Logger _logger = logger; public TSettings Resolve<TRepository, TSettings>(string implementationName) where TRepository : class where TSettings : IProviderConfig { dynamic dynamicRepo = _serviceProvider.Value.GetRequiredService<TRepository>(); IEnumerable<ProviderDefinition> definitions = dynamicRepo.All(); ProviderDefinition? definition = definitions.FirstOrDefault(d => d.Implementation == implementationName); if (definition?.Settings is TSettings settings) { _logger.Debug($"Resolved settings for {implementationName} from {typeof(TRepository).Name}"); return settings; } throw new InvalidOperationException($"Settings not found for {implementationName}. Ensure it's configured and enabled."); } } } ================================================ FILE: Tubifarry/Core/Utilities/UserAgentValidator.cs ================================================ using System.Text.RegularExpressions; namespace Tubifarry.Core.Utilities { /// <summary> /// Interface for User-Agent validation and parsing /// </summary> public interface IUserAgentValidator { /// <summary> /// Validates if a User-Agent string is allowed /// </summary> /// <param name="userAgent">User-Agent string to validate</param> /// <returns>True if allowed, otherwise false</returns> bool IsAllowed(string userAgent); /// <summary> /// Parses User-Agent into product components< /// /summary> /// <param name="userAgent">User-Agent string to parse</param> /// <returns>Collection of product tokens</returns> IEnumerable<UserAgentProduct> Parse(string userAgent); /// <summary> /// Adds pattern to allowlist /// </summary> /// <param name="pattern">Pattern to allow</param> void AddAllowedPattern(string pattern); /// <summary> /// Adds pattern to blacklist /// </summary> /// <param name="pattern">Pattern to block</param> void AddBlacklistPattern(string pattern); } /// <summary> /// Represents product token in User-Agent string /// </summary> public record UserAgentProduct(string Name, string Version) { public override string ToString() => Version != null ? $"{Name}/{Version}" : Name; } /// <summary> /// Validates User-Agents against allow/block lists /// </summary> public partial class UserAgentValidator : IUserAgentValidator { private readonly HashSet<string> _allowedExact = new(StringComparer.OrdinalIgnoreCase); private readonly List<Regex> _allowedRegex = []; private readonly HashSet<string> _blackExact = new(StringComparer.OrdinalIgnoreCase); private readonly List<Regex> _blackRegex = []; public static UserAgentValidator Instance { get; private set; } = new(); /// <summary> /// Creates validator with optional initial patterns /// </summary> /// <param name="allowed">Initial allow patterns</param> /// <param name="blacklisted">Initial block patterns</param> public UserAgentValidator(IEnumerable<string>? allowed = null, IEnumerable<string>? blacklisted = null) { allowed?.ToList().ForEach(AddAllowedPattern); blacklisted?.ToList().ForEach(AddBlacklistPattern); Instance = this; } /// <inheritdoc/> public bool IsAllowed(string userAgent) { if (string.IsNullOrWhiteSpace(userAgent)) return false; if (!IsValidFormat(userAgent)) return false; if (_blackExact.Contains(userAgent)) return false; if (_blackRegex.Any(p => p.IsMatch(userAgent))) return false; if (_allowedExact.Count == 0 && _allowedRegex.Count == 0) return true; return _allowedExact.Contains(userAgent) || _allowedRegex.Any(p => p.IsMatch(userAgent)); } /// <inheritdoc/> public IEnumerable<UserAgentProduct> Parse(string userAgent) => userAgent.Split(' ') .Select(t => t.Split(['/'], 2)) .Where(p => p.Length > 0 && !string.IsNullOrWhiteSpace(p[0])) .Select(p => new UserAgentProduct(p[0], p.Length > 1 ? p[1] : string.Empty)); /// <inheritdoc/> public void AddAllowedPattern(string pattern) => AddPattern(pattern, _allowedExact, _allowedRegex); /// <inheritdoc/> public void AddBlacklistPattern(string pattern) => AddPattern(pattern, _blackExact, _blackRegex); private bool IsValidFormat(string ua) { try { return Parse(ua).All(p => TokenPattern().IsMatch(p.Name) && (p.Version == null || TokenPattern().IsMatch(p.Version))); } catch { return false; } } /// <summary> /// Adds a pattern to either the exact matches or regex patterns collection /// </summary> private static void AddPattern(string p, HashSet<string> exact, List<Regex> regex) { if (string.IsNullOrWhiteSpace(p)) throw new ArgumentException("Pattern required", nameof(p)); if (p.IndexOfAny(['*', '?', '+', '(', '[', '\\', '.']) >= 0) { try { regex.Add(new Regex(p, RegexOptions.IgnoreCase | RegexOptions.Compiled)); } catch { exact.Add(p); } } else { exact.Add(p); } } [GeneratedRegex(@"^[!#$%&'*+\-.0-9A-Z^_`a-z|~]+$")] private static partial Regex TokenPattern(); } } ================================================ FILE: Tubifarry/Debug.targets ================================================ <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <!-- Copy the files to the plugin folder --> <Target Name="PostBuild" AfterTargets="ILRepacker" Condition="'$(Configuration)' == 'Debug'"> <PropertyGroup> <PluginName>$(MSBuildProjectName)</PluginName> <DestinationFolder>C:\ProgramData\Lidarr\plugins\$(Author)\$(PluginName)</DestinationFolder> </PropertyGroup> <Message Text="Plugin Name: $(PluginName)" Importance="high" /> <Message Text="Destination Folder: $(DestinationFolder)" Importance="high" /> <MakeDir Directories="$(DestinationFolder)" /> <Copy SourceFiles="$(TargetPath)" DestinationFiles="$(DestinationFolder)\Lidarr.Plugin.$(PluginName).dll" SkipUnchangedFiles="true" /> <Copy SourceFiles="$(TargetDir)$(TargetName).pdb" DestinationFiles="$(DestinationFolder)\Lidarr.Plugin.$(PluginName).pdb" SkipUnchangedFiles="true" Condition="Exists('$(TargetDir)$(TargetName).pdb')" /> <Copy SourceFiles="$(TargetDir)$(TargetName).deps.json" DestinationFiles="$(DestinationFolder)\Lidarr.Plugin.$(PluginName).deps.json" SkipUnchangedFiles="true" Condition="Exists('$(TargetDir)$(TargetName).deps.json')" /> </Target> </Project> ================================================ FILE: Tubifarry/Download/Base/BaseDownloadManager.cs ================================================ using NLog; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Requests; namespace Tubifarry.Download.Base { /// <summary> /// Generic interface for download managers /// </summary> public interface IBaseDownloadManager<TDownloadRequest, TOptions, TClient> where TDownloadRequest : BaseDownloadRequest<TOptions> where TOptions : BaseDownloadOptions, new() { Task<string> Download(RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, TClient provider); IEnumerable<DownloadClientItem> GetItems(); void RemoveItem(DownloadClientItem item); } /// <summary> /// Generic base download manager implementation with common functionality /// </summary> public abstract class BaseDownloadManager<TDownloadRequest, TOptions, TClient>(Logger logger) : IBaseDownloadManager<TDownloadRequest, TOptions, TClient> where TDownloadRequest : BaseDownloadRequest<TOptions> where TOptions : BaseDownloadOptions, new() { private readonly RequestContainer<TDownloadRequest> _queue = []; protected readonly Logger _logger = logger; protected readonly RequestHandler _requesthandler = []; /// <summary> /// Factory method to create download request instances /// </summary> protected abstract Task<TDownloadRequest> CreateDownloadRequest(RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, TClient provider); /// <summary> /// Common implementation for downloading with error handling /// </summary> public virtual async Task<string> Download(RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, TClient provider) { try { TDownloadRequest downloadRequest = await CreateDownloadRequest(remoteAlbum, indexer, namingConfig, provider); _queue.Add(downloadRequest); _logger.Debug($"Added download: {downloadRequest.ID} | {remoteAlbum.Release.Title}"); return downloadRequest.ID; } catch (Exception ex) { _logger.Error(ex, $"Error adding download for album: {remoteAlbum.Release.Title}"); throw; } } /// <summary> /// Common implementation for getting all download items /// </summary> public virtual IEnumerable<DownloadClientItem> GetItems() => _queue.Select(x => x.ClientItem); /// <summary> /// Common implementation for removing download items with proper error handling /// </summary> public virtual void RemoveItem(DownloadClientItem item) { try { TDownloadRequest? request = _queue.ToList().Find(x => x.ID == item.DownloadId); if (request == null) { _logger.Warn($"Attempted to remove non-existent download item: {item.DownloadId}"); return; } request.Dispose(); _queue.Remove(request); _logger.Debug($"Removed download: {item.DownloadId}"); } catch (Exception ex) { _logger.Error(ex, $"Error removing download item: {item.DownloadId}"); } } } } ================================================ FILE: Tubifarry/Download/Base/BaseDownloadOptions.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Organizer; using Requests.Options; namespace Tubifarry.Download.Base { /// <summary> /// Base options for download requests containing common configuration /// </summary> public record BaseDownloadOptions : RequestOptions<string, string> { /// <summary> /// Client info for tracking the download in Lidarr /// </summary> public DownloadClientItemClientInfo? ClientInfo { get; set; } /// <summary> /// Path where downloads will be stored /// </summary> public string DownloadPath { get; set; } = string.Empty; /// <summary> /// Base URL of the service instance /// </summary> public string BaseUrl { get; set; } = string.Empty; /// <summary> /// Timeout for HTTP requests in seconds /// </summary> public int RequestTimeout { get; set; } = 60; /// <summary> /// Maximum download speed in bytes per second (0 = unlimited) /// </summary> public int MaxDownloadSpeed { get; set; } /// <summary> /// Number of chunks for download /// </summary> public int Chunks { get; set; } = 1; /// <summary> /// Number of times to retry connections /// </summary> public int ConnectionRetries { get; set; } = 3; /// <summary> /// Naming configuration from Lidarr /// </summary> public NamingConfig? NamingConfig { get; set; } /// <summary> /// Whether this download is for a track (true) or album (false) /// </summary> public bool IsTrack { get; set; } /// <summary> /// The item ID/URL to download from /// </summary> public string ItemId { get; set; } = string.Empty; /// <summary> /// Request tnterceptors for requests /// </summary> public IEnumerable<IHttpRequestInterceptor> RequestInterceptors { get; set; } = []; public BaseDownloadOptions() { } protected BaseDownloadOptions(BaseDownloadOptions options) : base(options) { ClientInfo = options.ClientInfo; DownloadPath = options.DownloadPath; BaseUrl = options.BaseUrl; RequestTimeout = options.RequestTimeout; MaxDownloadSpeed = options.MaxDownloadSpeed; ConnectionRetries = options.ConnectionRetries; NamingConfig = options.NamingConfig; IsTrack = options.IsTrack; Chunks = options.Chunks; ItemId = options.ItemId; RequestInterceptors = options.RequestInterceptors; } } } ================================================ FILE: Tubifarry/Download/Base/BaseDownloadRequest.cs ================================================ using DownloadAssistant.Requests; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Download; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using System.Text; using System.Text.RegularExpressions; using Tubifarry.Core.Utilities; namespace Tubifarry.Download.Base { /// <summary> /// Base class for download requests containing common functionality /// </summary> public abstract partial class BaseDownloadRequest<TOptions> : Request<TOptions, string, string> where TOptions : BaseDownloadOptions, new() { protected readonly OsPath _destinationPath; protected readonly StringBuilder _message = new(); protected readonly RequestContainer<IRequest> _requestContainer = []; protected readonly RequestContainer<LoadRequest> _trackContainer = []; protected readonly RemoteAlbum _remoteAlbum; protected readonly Album _albumData; protected readonly DownloadClientItem _clientItem; protected readonly ReleaseFormatter _releaseFormatter; protected readonly Logger _logger; protected int _expectedTrackCount; protected byte[]? _albumCover; // Progress tracking private DateTime _lastUpdateTime = DateTime.MinValue; private long _lastRemainingSize; protected ReleaseInfo ReleaseInfo => _remoteAlbum.Release; public override Task Task => _requestContainer.Task; public override RequestState State => _requestContainer.State; public string ID { get; } = Guid.NewGuid().ToString(); public virtual DownloadClientItem ClientItem { get { long remainingSize = GetRemainingSize(); long totalDownloaded = _trackContainer.Sum(t => t.BytesDownloaded); long estimatedTotalSize = totalDownloaded + remainingSize; _clientItem.TotalSize = Math.Max(_clientItem.TotalSize, estimatedTotalSize); _clientItem.RemainingSize = remainingSize; _clientItem.Status = GetDownloadItemStatus(); _clientItem.RemainingTime = GetRemainingTime(); _clientItem.Message = GetDistinctMessages(); _clientItem.CanBeRemoved = HasCompleted(); _clientItem.CanMoveFiles = HasCompleted(); return _clientItem; } } protected BaseDownloadRequest(RemoteAlbum remoteAlbum, TOptions? options) : base(options) { _logger = NzbDroneLogger.GetLogger(this); _remoteAlbum = remoteAlbum; _albumData = remoteAlbum.Albums.FirstOrDefault() ?? new Album(); _releaseFormatter = new ReleaseFormatter(ReleaseInfo, remoteAlbum.Artist, Options.NamingConfig); _requestContainer.Add(_trackContainer); _expectedTrackCount = Options.IsTrack ? 1 : remoteAlbum.Albums.FirstOrDefault()?.AlbumReleases.Value?.FirstOrDefault()?.TrackCount ?? 0; _destinationPath = new OsPath(Path.Combine( Options.DownloadPath, _releaseFormatter.BuildArtistFolderName(null), _releaseFormatter.BuildAlbumFilename("{Album Title}", new Album() { Title = GetAlbumTitle() }) )); _clientItem = CreateClientItem(); _logger.Debug($"Processing download. Type: {(Options.IsTrack ? "track" : "album")}, ID: {Options.ItemId}"); } /// <summary> /// Implement the main download processing logic /// </summary> protected abstract Task ProcessDownloadAsync(CancellationToken token); /// <summary> /// Get the album title for folder naming /// </summary> protected virtual string GetAlbumTitle() => ReleaseInfo.Album ?? ReleaseInfo.Title; /// <summary> /// Sanitizes a filename by removing invalid characters /// </summary> protected static string SanitizeFileName(string fileName) => string.IsNullOrEmpty(fileName) ? "Unknown" : FileNameSanitizerRegex().Replace(fileName, "_").Trim(); /// <summary> /// Logs a message and appends it to the client message buffer /// </summary> protected void LogAndAppendMessage(string message, LogLevel logLevel) { _message.AppendLine(message); _logger?.Log(logLevel, message); } /// <summary> /// Creates the initial download client item /// </summary> protected virtual DownloadClientItem CreateClientItem() => new() { DownloadId = ID, Title = ReleaseInfo.Title, TotalSize = ReleaseInfo.Size, DownloadClientInfo = Options.ClientInfo, OutputPath = _destinationPath, }; /// <summary> /// Calculates remaining download time based on current progress /// </summary> protected virtual TimeSpan? GetRemainingTime() { long remainingSize = GetRemainingSize(); if (_lastUpdateTime != DateTime.MinValue && _lastRemainingSize != 0) { TimeSpan timeElapsed = DateTime.UtcNow - _lastUpdateTime; long bytesDownloaded = _lastRemainingSize - remainingSize; if (timeElapsed.TotalSeconds > 0 && bytesDownloaded > 0) { double bytesPerSecond = bytesDownloaded / timeElapsed.TotalSeconds; double remainingSeconds = remainingSize / bytesPerSecond; return remainingSeconds < 0 ? TimeSpan.FromSeconds(10) : TimeSpan.FromSeconds(remainingSeconds); } } _lastUpdateTime = DateTime.UtcNow; _lastRemainingSize = remainingSize; return null; } /// <summary> /// Gets distinct messages from the message buffer /// </summary> protected virtual string GetDistinctMessages() => string.Join(Environment.NewLine, _message.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Distinct()); /// <summary> /// Calculates remaining download size based on track progress /// </summary> protected virtual long GetRemainingSize() { long totalDownloaded = _trackContainer.Sum(t => t.BytesDownloaded); IEnumerable<LoadRequest> knownSizes = _trackContainer.Where(t => t.ContentLength > 0); int knownCount = knownSizes.Count(); return (_expectedTrackCount, knownCount) switch { (0, _) => ReleaseInfo.Size - totalDownloaded, (var expected, var count) when count == expected => knownSizes.Sum(t => t.ContentLength) - totalDownloaded, (var expected, var count) when count > 2 => Math.Max(0, Math.Max((long)(knownSizes.Average(t => t.ContentLength) * expected), ReleaseInfo.Size) - totalDownloaded), (var expected, var count) when count > 0 => Math.Max((long)(knownSizes.Average(t => t.ContentLength) * expected), ReleaseInfo.Size) - totalDownloaded, _ => ReleaseInfo.Size - totalDownloaded }; } /// <summary> /// Maps request state to download item status /// </summary> public virtual DownloadItemStatus GetDownloadItemStatus() => State switch { RequestState.Idle => DownloadItemStatus.Queued, RequestState.Paused => DownloadItemStatus.Paused, RequestState.Running => DownloadItemStatus.Downloading, RequestState.Compleated => DownloadItemStatus.Completed, RequestState.Failed => _requestContainer.Count(x => x.State == RequestState.Failed) >= _requestContainer.Count / 2 ? DownloadItemStatus.Failed : _requestContainer.All(x => x.HasCompleted()) ? DownloadItemStatus.Completed : DownloadItemStatus.Failed, _ => DownloadItemStatus.Warning, }; /// <summary> /// Builds a track filename using the release formatter /// </summary> protected string BuildTrackFilename(Track track, Album album, string extension = ".flac") => _releaseFormatter.BuildTrackFilename(null, track, album) + extension; public override void Start() => throw new NotImplementedException(); public override void Pause() => throw new NotImplementedException(); protected override Task<RequestReturn> RunRequestAsync() => throw new NotImplementedException(); [GeneratedRegex(@"[\\/:\*\?""<>\|]", RegexOptions.Compiled)] private static partial Regex FileNameSanitizerRegex(); } } ================================================ FILE: Tubifarry/Download/Base/BaseHttpClient.cs ================================================ using DownloadAssistant.Base; using NzbDrone.Common.Http; namespace Tubifarry.Download.Base { /// <summary> /// HTTP client wrapper for download operations /// Provides standardized HTTP operations with proper headers and error handling /// Never modifies the shared HttpClient uses individual requests with proper headers /// </summary> public class BaseHttpClient { private readonly System.Net.Http.HttpClient _httpClient = HttpGet.HttpClient; private readonly TimeSpan _timeout; private readonly List<IHttpRequestInterceptor> _requestInterceptors; /// <summary> /// Initializes a new instance of the BaseHttpClient /// </summary> /// <param name="baseUrl">Base URL for the service instance</param> /// <param name="requestInterceptors">Optional list of request interceptors</param> /// <param name="timeout">Request timeout (default: 60 seconds)</param> public BaseHttpClient(string baseUrl, IEnumerable<IHttpRequestInterceptor> requestInterceptors, TimeSpan? timeout = null) { BaseUrl = baseUrl?.TrimEnd('/') ?? throw new ArgumentNullException(nameof(baseUrl)); _timeout = timeout ?? TimeSpan.FromSeconds(60); _requestInterceptors = requestInterceptors?.ToList() ?? []; } /// <summary> /// Gets the base URL for this service instance /// </summary> public string BaseUrl { get; } /// <summary> /// Creates a properly configured HttpRequestMessage with standard headers /// </summary> /// <param name="method">HTTP method</param> /// <param name="url">The URL to request (can be relative or absolute)</param> /// <returns>Configured HttpRequestMessage</returns> public HttpRequestMessage CreateRequest(HttpMethod method, string url) { string requestUrl = url.StartsWith("http") ? url : new Uri(new Uri(BaseUrl), url).ToString(); HttpRequestMessage request = new(method, requestUrl); request.Headers.Add("User-Agent", Tubifarry.UserAgent); request.Headers.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); return request; } /// <summary> /// Sends an HTTP request with timeout handling and interceptor support /// </summary> /// <param name="request">The HTTP request to send</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>HTTP response message</returns> /// <exception cref="Exception">Thrown when the request fails</exception> public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) { try { HttpRequest nzbRequest = ConvertToNzbRequest(request); foreach (IHttpRequestInterceptor interceptor in _requestInterceptors) { nzbRequest = interceptor.PreRequest(nzbRequest); } ApplyNzbRequestToHttpRequestMessage(nzbRequest, request); using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_timeout); HttpResponseMessage response = await _httpClient.SendAsync(request, cts.Token); HttpResponse nzbResponse = await ConvertToNzbResponse(nzbRequest, response); foreach (IHttpRequestInterceptor interceptor in _requestInterceptors) { nzbResponse = interceptor.PostResponse(nzbResponse); } return ConvertToHttpResponseMessage(nzbResponse); } catch (Exception ex) { throw new Exception($"HTTP request failed for URL '{request.RequestUri}': {ex.Message}", ex); } } private static HttpRequest ConvertToNzbRequest(HttpRequestMessage httpRequest) { HttpRequest nzbRequest = new(httpRequest.RequestUri?.ToString()) { Method = httpRequest.Method }; List<string> requestHeaderKeys = new(); List<string> contentHeaderKeys = new(); string allRequestHeaders = httpRequest.Headers.ToString(); foreach (string line in allRequestHeaders.Split(new[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries)) { int colonIndex = line.IndexOf(':'); if (colonIndex > 0) { string headerName = line.Substring(0, colonIndex).Trim(); string headerValue = line.Substring(colonIndex + 1).Trim(); requestHeaderKeys.Add(headerName); nzbRequest.Headers[headerName] = headerValue; } } if (httpRequest.Content != null) { string allContentHeaders = httpRequest.Content.Headers.ToString(); foreach (string line in allContentHeaders.Split(new[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries)) { int colonIndex = line.IndexOf(':'); if (colonIndex > 0) { string headerName = line.Substring(0, colonIndex).Trim(); string headerValue = line.Substring(colonIndex + 1).Trim(); contentHeaderKeys.Add(headerName); nzbRequest.Headers[headerName] = headerValue; } } byte[] contentBytes = httpRequest.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); if (contentBytes.Length > 0) { nzbRequest.SetContent(contentBytes); } } nzbRequest.Cookies["__REQUEST_HEADERS__"] = string.Join("|", requestHeaderKeys); nzbRequest.Cookies["__CONTENT_HEADERS__"] = string.Join("|", contentHeaderKeys); return nzbRequest; } private static void ApplyNzbRequestToHttpRequestMessage(HttpRequest nzbRequest, HttpRequestMessage httpRequest) { HashSet<string> originalRequestHeaders = nzbRequest.Cookies.TryGetValue("__REQUEST_HEADERS__", out string? valueH) ? new HashSet<string>(valueH.Split('|', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase) : new HashSet<string>(StringComparer.OrdinalIgnoreCase); HashSet<string> originalContentHeaders = nzbRequest.Cookies.TryGetValue("__CONTENT_HEADERS__", out string? valueC) ? new HashSet<string>(valueC.Split('|', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase) : new HashSet<string>(StringComparer.OrdinalIgnoreCase); foreach ((string key, string value) in nzbRequest.Headers) { if (originalRequestHeaders.Contains(key)) { httpRequest.Headers.Remove(key); httpRequest.Headers.TryAddWithoutValidation(key, value); } else if (originalContentHeaders.Contains(key)) { if (httpRequest.Content != null) { httpRequest.Content.Headers.Remove(key); httpRequest.Content.Headers.TryAddWithoutValidation(key, value); } } else { httpRequest.Headers.TryAddWithoutValidation(key, value); } } List<KeyValuePair<string, string>> actualCookies = nzbRequest.Cookies .Where(c => !c.Key.StartsWith("__") || !c.Key.EndsWith("__")) .ToList(); if (actualCookies.Count > 0) { string cookieHeader = string.Join("; ", actualCookies.Select(c => $"{c.Key}={c.Value}")); httpRequest.Headers.Remove("Cookie"); httpRequest.Headers.Add("Cookie", cookieHeader); } } private static async Task<HttpResponse> ConvertToNzbResponse(HttpRequest nzbRequest, HttpResponseMessage httpResponse) { HttpHeader headers = []; foreach (KeyValuePair<string, IEnumerable<string>> header in httpResponse.Headers) { headers[header.Key] = string.Join(", ", header.Value); } if (httpResponse.Content != null) { foreach (KeyValuePair<string, IEnumerable<string>> header in httpResponse.Content.Headers) { headers[header.Key] = string.Join(", ", header.Value); } } byte[] responseData = []; if (httpResponse.Content != null) { responseData = await httpResponse.Content.ReadAsByteArrayAsync(); } return new HttpResponse(nzbRequest, headers, responseData, httpResponse.StatusCode, httpResponse.Version); } private static HttpResponseMessage ConvertToHttpResponseMessage(HttpResponse nzbResponse) { HttpResponseMessage httpResponse = new(nzbResponse.StatusCode) { Version = nzbResponse.Version, Content = new ByteArrayContent(nzbResponse.ResponseData ?? []) }; foreach (KeyValuePair<string, string> header in nzbResponse.Headers) { if (!httpResponse.Headers.TryAddWithoutValidation(header.Key, header.Value)) { httpResponse.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); } } return httpResponse; } /// <summary> /// Performs a GET request and returns the response as a string /// </summary> /// <param name="url">The URL to request (can be relative or absolute)</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>Response content as string</returns> /// <exception cref="Exception">Thrown when the request fails</exception> public async Task<string> GetStringAsync(string url, CancellationToken cancellationToken = default) { using HttpRequestMessage request = CreateRequest(HttpMethod.Get, url); using HttpResponseMessage response = await SendAsync(request, cancellationToken); response.EnsureSuccessStatusCode(); using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_timeout); return await response.Content.ReadAsStringAsync(cts.Token); } /// <summary> /// Performs a GET request and returns the HttpResponseMessage /// </summary> /// <param name="url">The URL to request (can be relative or absolute)</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>HTTP response message</returns> /// <exception cref="Exception">Thrown when the request fails</exception> public async Task<HttpResponseMessage> GetAsync(string url, CancellationToken cancellationToken = default) { try { using HttpRequestMessage request = CreateRequest(HttpMethod.Get, url); return await SendAsync(request, cancellationToken); } catch (Exception ex) { throw new Exception($"HTTP GET request failed for URL '{url}': {ex.Message}", ex); } } /// <summary> /// Performs a POST request with the provided HTTP request message /// Adds standard headers to the request if not already present /// </summary> /// <param name="request">The HTTP request message to send</param> /// <returns>HTTP response message</returns> /// <exception cref="Exception">Thrown when the request fails</exception> public async Task<HttpResponseMessage> PostAsync(HttpRequestMessage request) { try { if (!request.Headers.Contains("User-Agent")) request.Headers.Add("User-Agent", Tubifarry.UserAgent); if (!request.Headers.Contains("Accept")) request.Headers.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); return await SendAsync(request); } catch (Exception ex) { throw new Exception($"HTTP POST request failed for URL '{request.RequestUri}': {ex.Message}", ex); } } /// <summary> /// Performs a POST request with string content /// </summary> /// <param name="url">The URL to post to (can be relative or absolute)</param> /// <param name="content">The content to post</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>HTTP response message</returns> /// <exception cref="Exception">Thrown when the request fails</exception> public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, CancellationToken cancellationToken = default) { try { using HttpRequestMessage request = CreateRequest(HttpMethod.Post, url); request.Content = content; return await SendAsync(request, cancellationToken); } catch (Exception ex) { throw new Exception($"HTTP POST request failed for URL '{url}': {ex.Message}", ex); } } } } ================================================ FILE: Tubifarry/Download/Clients/DABMusic/DABMusicClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using System.Text.RegularExpressions; using Tubifarry.Download.Base; using Tubifarry.Indexers.DABMusic; namespace Tubifarry.Download.Clients.DABMusic { /// <summary> /// DABMusic download client for high-quality music downloads /// Integrates with DABMusic API to download tracks and albums /// </summary> public class DABMusicClient : DownloadClientBase<DABMusicProviderSettings> { private readonly IDABMusicDownloadManager _downloadManager; private readonly INamingConfigService _namingService; private readonly IDABMusicSessionManager _sessionManager; private readonly IEnumerable<IHttpRequestInterceptor> _requestInterceptors; public DABMusicClient( IDABMusicDownloadManager downloadManager, IDABMusicSessionManager sessionManager, IConfigService configService, IDiskProvider diskProvider, INamingConfigService namingConfigService, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, IEnumerable<IHttpRequestInterceptor> requestInterceptors, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _downloadManager = downloadManager; _requestInterceptors = requestInterceptors; _namingService = namingConfigService; _sessionManager = sessionManager; } public override string Name => "DABMusic"; public override string Protocol => nameof(QobuzDownloadProtocol); public new DABMusicProviderSettings Settings => base.Settings; public override Task<string> Download(RemoteAlbum remoteAlbum, IIndexer indexer) => _downloadManager.Download(remoteAlbum, indexer, _namingService.GetConfig(), this); public override IEnumerable<DownloadClientItem> GetItems() => _downloadManager.GetItems(); public override void RemoveItem(DownloadClientItem item, bool deleteData) { if (deleteData) DeleteItemData(item); _downloadManager.RemoveItem(item); } public override DownloadClientInfo GetStatus() => new() { IsLocalhost = false, OutputRootFolders = [new OsPath(Settings.DownloadPath)] }; protected override void Test(List<ValidationFailure> failures) { if (!_diskProvider.FolderExists(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path does not exist")); return; } if (!_diskProvider.FolderWritable(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path is not writable")); } try { BaseHttpClient httpClient = new(Settings.BaseUrl, _requestInterceptors, TimeSpan.FromSeconds(30)); string response = httpClient.GetStringAsync("/").GetAwaiter().GetResult(); if (string.IsNullOrEmpty(response)) { failures.Add(new ValidationFailure("BaseUrl", "Cannot connect to DABMusic instance: Empty response")); return; } if (!response.Contains("DABMusic", StringComparison.OrdinalIgnoreCase) && !response.Contains("dabmusic", StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(response, "<title>.*?(DAB|Music).*?", RegexOptions.IgnoreCase)) { failures.Add(new ValidationFailure("BaseUrl", "The provided URL does not appear to be a DABMusic instance")); return; } if (!string.IsNullOrWhiteSpace(Settings.Email) && !string.IsNullOrWhiteSpace(Settings.Password)) { DABMusicSession? session = _sessionManager.GetOrCreateSession(Settings.BaseUrl.Trim(), Settings.Email, Settings.Password, true); if (session == null) { failures.Add(new ValidationFailure("Email", "Failed to authenticate with DABMusic. Check your email and password.")); return; } _logger.Debug($"Successfully authenticated with DABMusic as {Settings.Email}"); } } catch (Exception ex) { _logger.Error(ex, "Error connecting to DABMusic instance"); failures.Add(new ValidationFailure("BaseUrl", $"Cannot connect to DABMusic instance: {ex.Message}")); } } } } ================================================ FILE: Tubifarry/Download/Clients/DABMusic/DABMusicDownloadManager.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Tubifarry.Download.Base; using Tubifarry.Indexers.DABMusic; namespace Tubifarry.Download.Clients.DABMusic { public interface IDABMusicDownloadManager : IBaseDownloadManager { } public class DABMusicDownloadManager : BaseDownloadManager, IDABMusicDownloadManager { private readonly IDABMusicSessionManager _sessionManager; private readonly IEnumerable _requestInterceptors; public DABMusicDownloadManager(IDABMusicSessionManager sessionManager, IEnumerable requestInterceptors, Logger logger) : base(logger) { _sessionManager = sessionManager; _requestInterceptors = requestInterceptors; } protected override async Task CreateDownloadRequest( RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, DABMusicClient provider) { string baseUrl = provider.Settings.BaseUrl; bool isTrack = remoteAlbum.Release.DownloadUrl.Contains("/track/"); string itemId = remoteAlbum.Release.DownloadUrl.Split('/').Last(); _logger.Trace($"Type from URL: {(isTrack ? "Track" : "Album")}, Extracted ID: {itemId}"); DABMusicDownloadOptions options = new() { Handler = _requesthandler, DownloadPath = provider.Settings.DownloadPath, BaseUrl = baseUrl, MaxDownloadSpeed = provider.Settings.MaxDownloadSpeed * 1024, // Convert KB/s to bytes/s ConnectionRetries = provider.Settings.ConnectionRetries, NamingConfig = namingConfig, RequestInterceptors = _requestInterceptors, DelayBetweenAttemps = TimeSpan.FromSeconds(2), NumberOfAttempts = (byte)provider.Settings.ConnectionRetries, ClientInfo = DownloadClientItemClientInfo.FromDownloadClient(provider, false), IsTrack = isTrack, ItemId = itemId, Email = provider.Settings.Email, Password = provider.Settings.Password }; _requesthandler.MaxParallelism = provider.Settings.MaxParallelDownloads; return new DABMusicDownloadRequest(remoteAlbum, _sessionManager, options); } } } ================================================ FILE: Tubifarry/Download/Clients/DABMusic/DABMusicDownloadOptions.cs ================================================ using Tubifarry.Download.Base; namespace Tubifarry.Download.Clients.DABMusic { public record DABMusicDownloadOptions : BaseDownloadOptions { public string Email { get; set; } = string.Empty; public string Password { get; set; } = string.Empty; public DABMusicDownloadOptions() : base() { } protected DABMusicDownloadOptions(DABMusicDownloadOptions options) : base(options) { Email = options.Email; Password = options.Password; } } } ================================================ FILE: Tubifarry/Download/Clients/DABMusic/DABMusicDownloadRequest.cs ================================================ using DownloadAssistant.Options; using DownloadAssistant.Requests; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Download.Base; using Tubifarry.Indexers.DABMusic; namespace Tubifarry.Download.Clients.DABMusic { /// /// DABMusic download request handling track and album downloads /// public class DABMusicDownloadRequest : BaseDownloadRequest { private readonly BaseHttpClient _httpClient; private readonly IDABMusicSessionManager _sessionManager; private DABMusicAlbum? _currentAlbum; private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString }; public DABMusicDownloadRequest(RemoteAlbum remoteAlbum, IDABMusicSessionManager sessionManager, DABMusicDownloadOptions? options) : base(remoteAlbum, options) { _httpClient = new BaseHttpClient(Options.BaseUrl, Options.RequestInterceptors, TimeSpan.FromSeconds(Options.RequestTimeout)); _sessionManager = sessionManager; _requestContainer.Add(new OwnRequest(async (token) => { try { await ProcessDownloadAsync(token); return true; } catch (Exception ex) { LogAndAppendMessage($"Error processing download: {ex.Message}", LogLevel.Error); throw; } }, new RequestOptions() { CancellationToken = Token, DelayBetweenAttemps = Options.DelayBetweenAttemps, NumberOfAttempts = Options.NumberOfAttempts, Priority = RequestPriority.Low, Handler = Options.Handler })); } protected override async Task ProcessDownloadAsync(CancellationToken token) { _logger.Trace($"Processing {(Options.IsTrack ? "track" : "album")}: {ReleaseInfo.Title}"); if (Options.IsTrack) await ProcessSingleTrackAsync(Options.ItemId, token); else await ProcessAlbumAsync(Options.ItemId, token); } private async Task ProcessSingleTrackAsync(string trackId, CancellationToken token) { _logger.Trace($"Processing single track with ID: {trackId}"); DABMusicTrack track = await GetTrackAsync(trackId, token); string streamUrl = await GetStreamUrlAsync(trackId, token); if (string.IsNullOrEmpty(streamUrl)) throw new Exception("Failed to get stream URL for track"); _currentAlbum = new DABMusicAlbum( Id: track.AlbumId ?? "unknown", Title: track.AlbumTitle ?? ReleaseInfo.Album, Artist: track.Artist, ArtistId: track.ArtistId, Cover: track.Cover, ReleaseDate: track.ReleaseDate, Genre: track.Genre, TrackCount: 1, Label: track.Label ); await DownloadAlbumCoverAsync(_currentAlbum.Cover, token); Track trackMetadata = CreateTrackFromDABData(track, _currentAlbum); Album albumMetadata = CreateAlbumFromDABData(_currentAlbum); string fileName = BuildTrackFilename(trackMetadata, albumMetadata); InitiateDownload(streamUrl, fileName, track, token); _requestContainer.Add(_trackContainer); } private async Task ProcessAlbumAsync(string albumId, CancellationToken token) { _logger.Trace($"Processing album with ID: {albumId}"); _currentAlbum = await GetAlbumAsync(albumId, token); if ((_currentAlbum.Tracks?.Count ?? 0) == 0) throw new Exception("No tracks found in album"); _expectedTrackCount = _currentAlbum.Tracks!.Count; _logger.Trace($"Found {_currentAlbum.Tracks.Count} tracks in album: {_currentAlbum.Title}"); await DownloadAlbumCoverAsync(_currentAlbum.Cover, token); for (int i = 0; i < _currentAlbum.Tracks.Count; i++) { DABMusicTrack track = _currentAlbum.Tracks[i]; try { string streamUrl = await GetStreamUrlAsync(track.Id, token); if (string.IsNullOrEmpty(streamUrl)) { _logger.Warn($"No stream URL available for track: {track.Title}"); continue; } Track trackMetadata = CreateTrackFromDABData(track, _currentAlbum); Album albumMetadata = CreateAlbumFromDABData(_currentAlbum); string trackFileName = BuildTrackFilename(trackMetadata, albumMetadata); InitiateDownload(streamUrl, trackFileName, track, token); _logger.Trace($"Track {i + 1}/{_currentAlbum.Tracks.Count} queued: {track.Title}"); } catch (Exception ex) { LogAndAppendMessage($"Track {i + 1}/{_currentAlbum.Tracks.Count} failed: {track.Title} - {ex.Message}", LogLevel.Error); } } _requestContainer.Add(_trackContainer); } private async Task RequestAsync(string url, CancellationToken token) { using HttpRequestMessage request = _httpClient.CreateRequest(HttpMethod.Get, url); if (!string.IsNullOrWhiteSpace(Options.Email) && !string.IsNullOrWhiteSpace(Options.Password)) { DABMusicSession? session = _sessionManager.GetOrCreateSession(Options.BaseUrl, Options.Email, Options.Password); if (session?.IsValid == true) request.Headers.Add("Cookie", session.SessionCookie); } using HttpResponseMessage response = await _httpClient.SendAsync(request, token); response.EnsureSuccessStatusCode(); return await response.Content.ReadAsStringAsync(token); } private async Task GetTrackAsync(string trackId, CancellationToken token) { try { string url = $"/api/track?trackId={Uri.EscapeDataString(trackId)}"; string response = await RequestAsync(url, token); DABMusicTrack? track = JsonSerializer.Deserialize(response, JsonOptions); if (track != null) return track; throw new Exception("Failed to parse track response"); } catch (Exception ex) { throw new Exception($"Failed to get track details: {ex.Message}", ex); } } private async Task GetAlbumAsync(string albumId, CancellationToken token) { try { string url = $"/api/album?albumId={Uri.EscapeDataString(albumId)}"; string response = await RequestAsync(url, token); try { DABMusicAlbumDetailsResponse? result = JsonSerializer.Deserialize(response, JsonOptions); if (result?.Album != null) return result.Album; } catch (JsonException) { DABMusicAlbum? album = JsonSerializer.Deserialize(response, JsonOptions); if (album != null) return album; } throw new Exception("Failed to parse album response"); } catch (Exception ex) { throw new Exception($"Failed to get album details: {ex.Message}", ex); } } private async Task GetStreamUrlAsync(string trackId, CancellationToken token) { try { string url = $"/api/stream?trackId={Uri.EscapeDataString(trackId)}"; string response = await RequestAsync(url, token); DABMusicStreamResponse? result = JsonSerializer.Deserialize(response, JsonOptions); return result?.Url ?? string.Empty; } catch (Exception ex) { throw new Exception($"Failed to get stream URL: {ex.Message}", ex); } } private async Task DownloadAlbumCoverAsync(string? coverUrl, CancellationToken token) { if (string.IsNullOrEmpty(coverUrl)) return; try { using HttpResponseMessage response = await _httpClient.GetAsync(coverUrl, token); response.EnsureSuccessStatusCode(); _albumCover = await response.Content.ReadAsByteArrayAsync(token); _logger.Trace($"Downloaded album cover: {_albumCover.Length} bytes"); } catch (Exception ex) { _logger.Warn(ex, "Failed to download album cover"); _albumCover = null; } } private void InitiateDownload(string streamUrl, string fileName, DABMusicTrack track, CancellationToken token) { LoadRequest downloadRequest = new(streamUrl, new LoadRequestOptions() { CancellationToken = token, CreateSpeedReporter = true, SpeedReporterTimeout = 1000, Priority = RequestPriority.Normal, MaxBytesPerSecond = Options.MaxDownloadSpeed, DelayBetweenAttemps = Options.DelayBetweenAttemps, Filename = fileName, AutoStart = true, DestinationPath = _destinationPath.FullPath, Handler = Options.Handler, DeleteFilesOnFailure = true, RequestFailed = (_, __) => LogAndAppendMessage($"Download failed: {fileName}", LogLevel.Error), WriteMode = WriteMode.AppendOrTruncate, }); OwnRequest postProcessRequest = new((t) => PostProcessTrackAsync(track, downloadRequest, t), new RequestOptions() { AutoStart = false, Priority = RequestPriority.High, DelayBetweenAttemps = Options.DelayBetweenAttemps, Handler = Options.Handler, CancellationToken = token, RequestFailed = (_, __) => { LogAndAppendMessage($"Post-processing failed: {fileName}", LogLevel.Error); try { if (File.Exists(downloadRequest.Destination)) File.Delete(downloadRequest.Destination); } catch { } } }); downloadRequest.TrySetSubsequentRequest(postProcessRequest); postProcessRequest.TrySetIdle(); _trackContainer.Add(downloadRequest); _requestContainer.Add(postProcessRequest); } private async Task PostProcessTrackAsync(DABMusicTrack trackInfo, LoadRequest request, CancellationToken token) { string trackPath = request.Destination; await Task.Delay(100, token); if (!File.Exists(trackPath)) return false; try { AudioMetadataHandler audioData = new(trackPath) { AlbumCover = _albumCover }; Album album = CreateAlbumFromDABData(_currentAlbum); Track track = CreateTrackFromDABData(trackInfo, _currentAlbum); if (!audioData.TryEmbedMetadata(album, track)) { _logger.Warn($"Failed to embed metadata for: {Path.GetFileName(trackPath)}"); return false; } _logger.Trace($"Successfully processed track: {Path.GetFileName(trackPath)}"); return true; } catch (Exception ex) { LogAndAppendMessage($"Post-processing failed for {Path.GetFileName(trackPath)}: {ex.Message}", LogLevel.Error); return false; } } private Album CreateAlbumFromDABData(DABMusicAlbum? albumInfo) { string albumTitle = albumInfo?.Title ?? ReleaseInfo.Album ?? _remoteAlbum.Albums?.FirstOrDefault()?.Title ?? "Unknown Album"; string artistName = albumInfo?.Artist ?? ReleaseInfo.Artist ?? _remoteAlbum.Artist?.Name ?? "Unknown Artist"; DateTime releaseDate = ReleaseInfo.PublishDate; if (!string.IsNullOrEmpty(albumInfo?.ReleaseDate) && DateTime.TryParse(albumInfo.ReleaseDate, out DateTime parsedDate)) releaseDate = parsedDate; return new Album { Title = albumTitle, ReleaseDate = releaseDate, Artist = new LazyLoaded(new Artist { Name = artistName, }), AlbumReleases = new LazyLoaded>(new List { new() { TrackCount = albumInfo?.TrackCount ?? 0, Title = albumTitle, Label = !string.IsNullOrEmpty(albumInfo?.Label)? new(){ albumInfo.Label } : new(), } }), Genres = _remoteAlbum.Albums?.FirstOrDefault()?.Genres, }; } private Track CreateTrackFromDABData(DABMusicTrack trackInfo, DABMusicAlbum? albumInfo) => new() { Title = trackInfo.Title, TrackNumber = trackInfo.TrackNumber.ToString(), AbsoluteTrackNumber = trackInfo.TrackNumber, Duration = trackInfo.Duration * 1000, Artist = new LazyLoaded(new Artist { Name = trackInfo.Artist ?? albumInfo?.Artist ?? ReleaseInfo.Artist ?? _remoteAlbum.Artist?.Name, }) }; } } ================================================ FILE: Tubifarry/Download/Clients/DABMusic/DABMusicProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; namespace Tubifarry.Download.Clients.DABMusic { public class DABMusicProviderSettingsValidator : AbstractValidator { public DABMusicProviderSettingsValidator() { // Validate DownloadPath RuleFor(x => x.DownloadPath) .IsValidPath() .WithMessage("Download path must be a valid directory."); // Validate BaseUrl RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); // Validate Email RuleFor(x => x.Email) .EmailAddress().WithMessage("Must be a valid email address.") .When(x => !string.IsNullOrWhiteSpace(x.Email)); // Validate ConnectionRetries RuleFor(x => x.ConnectionRetries) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(10) .WithMessage("Connection retries must be between 1 and 10."); // Validate MaxParallelDownloads RuleFor(x => x.MaxParallelDownloads) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(5) .WithMessage("Max parallel downloads must be between 1 and 5."); // Validate MaxDownloadSpeed RuleFor(x => x.MaxDownloadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Max download speed must be greater than or equal to 0.") .LessThanOrEqualTo(100_000) .WithMessage("Max download speed must be less than or equal to 100 MB/s."); } } /// /// Configuration settings for the DABMusic download client /// public class DABMusicProviderSettings : IProviderConfig { private static readonly DABMusicProviderSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Directory where downloaded files will be saved")] public string DownloadPath { get; set; } = string.Empty; [FieldDefinition(1, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the DABMusic API instance", Placeholder = "https://dabmusic.xyz")] public string BaseUrl { get; set; } = "https://dabmusic.xyz"; [FieldDefinition(2, Label = "Email", Type = FieldType.Textbox, HelpText = "Your DABMusic account email (optional for now)")] public string Email { get; set; } = ""; [FieldDefinition(3, Label = "Password", Type = FieldType.Password, HelpText = "Your DABMusic account password (optional for now)", Privacy = PrivacyLevel.Password)] public string Password { get; set; } = ""; [FieldDefinition(4, Type = FieldType.Number, Label = "Connection Retries", HelpText = "Number of times to retry failed connections", Advanced = true)] public int ConnectionRetries { get; set; } = 3; [FieldDefinition(5, Type = FieldType.Number, Label = "Max Parallel Downloads", HelpText = "Maximum number of downloads that can run simultaneously")] public int MaxParallelDownloads { get; set; } = 2; [FieldDefinition(6, Label = "Max Download Speed", Type = FieldType.Number, HelpText = "Set to 0 for unlimited speed. Limits download speed per file.", Unit = "KB/s", Advanced = true)] public int MaxDownloadSpeed { get; set; } = 0; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/ILucidaRateLimiter.cs ================================================ namespace Tubifarry.Download.Clients.Lucida; public interface ILucidaRateLimiter { #region Constants public const string WORKER_MAUS = "maus"; public const string WORKER_HUND = "hund"; public const string WORKER_KATZE = "katze"; public const int COOLDOWN_SECONDS = 60; public const int MIN_REQUEST_DELAY_MS = 4000; public const int BUCKET_SIZE_ESTIMATE = 15; public const int MAX_CONCURRENT_PER_WORKER = 2; public const int MAX_WORKER_RETRIES = 3; public const int MAX_WAIT_RETRIES = 2; public const int WAIT_ON_ALL_BLOCKED_MS = 60_000; public const int POLLING_MIN_DELAY_MS = 500; public const int POLLING_MAX_CONCURRENT = 5; #endregion IReadOnlyCollection Workers { get; } Task WaitForAvailableWorkerAsync(CancellationToken cancellationToken); void MarkWorkerRateLimited(string workerName); void ReleaseWorker(string workerName); void EnsureWorkerRegistered(string workerName); bool IsRateLimitedResponse(string responseContent); IReadOnlyDictionary GetWorkerStates(); Task AcquirePollingSlotAsync(string workerName, CancellationToken cancellationToken); void ReleasePollingSlot(string workerName); } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; namespace Tubifarry.Download.Clients.Lucida { /// /// Lucida download client for high-quality music downloads /// Integrates with Lucida web interface to download tracks and albums /// public class LucidaClient : DownloadClientBase { private readonly ILucidaDownloadManager _downloadManager; private readonly INamingConfigService _namingService; private readonly ILucidaRateLimiter _rateLimiter; public LucidaClient( ILucidaDownloadManager downloadManager, ILucidaRateLimiter rateLimiter, IConfigService configService, IDiskProvider diskProvider, INamingConfigService namingConfigService, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _downloadManager = downloadManager; _namingService = namingConfigService; _rateLimiter = rateLimiter; } public override string Name => "Lucida"; public override string Protocol => nameof(LucidaDownloadProtocol); public new LucidaProviderSettings Settings => base.Settings; public override Task Download(RemoteAlbum remoteAlbum, IIndexer indexer) => _downloadManager.Download(remoteAlbum, indexer, _namingService.GetConfig(), this); public override IEnumerable GetItems() => _downloadManager.GetItems(); public override void RemoveItem(DownloadClientItem item, bool deleteData) { if (deleteData) DeleteItemData(item); _downloadManager.RemoveItem(item); } public override DownloadClientInfo GetStatus() => new() { IsLocalhost = false, OutputRootFolders = [new OsPath(Settings.DownloadPath)] }; protected override void Test(List failures) { if (!_diskProvider.FolderExists(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path does not exist")); return; } if (!_diskProvider.FolderWritable(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path is not writable")); return; } TestWorkerHealth(); } private void TestWorkerHealth() { try { IReadOnlyDictionary states = _rateLimiter.GetWorkerStates(); List> availableWorkers = states.Where(s => s.Value.IsAvailable).ToList(); List> rateLimitedWorkers = [.. states.Where(s => s.Value.RateLimitedUntil.HasValue && s.Value.RateLimitedUntil.Value > DateTime.UtcNow)]; if (availableWorkers.Count == 0) { _logger.Debug("No Lucida workers currently available, all may be rate limited"); foreach (KeyValuePair worker in rateLimitedWorkers) { TimeSpan remaining = worker.Value.RateLimitedUntil!.Value - DateTime.UtcNow; _logger.Debug($" Worker '{worker.Key}' rate limited for {remaining.TotalSeconds:F0}s more"); } } else { _logger.Debug($"Lucida workers available: {availableWorkers.Count}/{states.Count}"); foreach (KeyValuePair state in states) { _logger.Trace($" Worker '{state.Key}': available={state.Value.IsAvailable}, " + $"active={state.Value.ActiveRequests}"); } } } catch (Exception ex) { _logger.Debug(ex, "Failed to check Lucida worker health during test"); } } } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaDownloadManager.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Tubifarry.Download.Base; using Tubifarry.Indexers.Lucida; namespace Tubifarry.Download.Clients.Lucida { /// /// Interface for the Lucida download manager /// public interface ILucidaDownloadManager : IBaseDownloadManager; /// /// Lucida download manager using the base download manager implementation /// public class LucidaDownloadManager( Logger logger, IEnumerable requestInterceptors, ILucidaRateLimiter rateLimiter ) : BaseDownloadManager(logger), ILucidaDownloadManager { protected override async Task CreateDownloadRequest( RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, LucidaClient provider) { string itemUrl = remoteAlbum.Release.DownloadUrl; string baseUrl = ((LucidaIndexerSettings)indexer.Definition.Settings).BaseUrl; _logger.Trace($"Processing Lucida download URL: {itemUrl} on Instance: {baseUrl}"); bool isTrack = remoteAlbum.Release.Source == "track"; _logger.Trace($"Type from Source field: {remoteAlbum.Release.Source} -> {(isTrack ? "Track" : "Album")}"); LucidaDownloadOptions options = new() { Handler = _requesthandler, DownloadPath = provider.Settings.DownloadPath, BaseUrl = baseUrl, RequestTimeout = provider.Settings.RequestTimeout, MaxDownloadSpeed = provider.Settings.MaxDownloadSpeed * 1024, // Convert KB/s to bytes/s ConnectionRetries = provider.Settings.ConnectionRetries, NamingConfig = namingConfig, DelayBetweenAttemps = TimeSpan.FromSeconds(2), RequestInterceptors = requestInterceptors, NumberOfAttempts = (byte)provider.Settings.ConnectionRetries, ClientInfo = DownloadClientItemClientInfo.FromDownloadClient(provider, false), IsTrack = isTrack, ItemId = itemUrl, RateLimiter = rateLimiter }; _requesthandler.MaxParallelism = provider.Settings.MaxParallelDownloads; await Task.Yield(); return new LucidaDownloadRequest(remoteAlbum, options); } } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaDownloadOptions.cs ================================================ using Tubifarry.Download.Base; namespace Tubifarry.Download.Clients.Lucida { public record LucidaDownloadOptions : BaseDownloadOptions { public ILucidaRateLimiter RateLimiter { get; init; } = null!; public LucidaDownloadOptions() { } public LucidaDownloadOptions(LucidaDownloadOptions options) : base(options) { RateLimiter = options.RateLimiter; } } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaDownloadRequest.cs ================================================ using DownloadAssistant.Options; using DownloadAssistant.Requests; using NLog; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using System.Text; using System.Text.Json; using Tubifarry.Core.Utilities; using Tubifarry.Download.Base; using Tubifarry.Indexers.Lucida; namespace Tubifarry.Download.Clients.Lucida { /// /// Lucida download request handling track and album downloads /// public class LucidaDownloadRequest : BaseDownloadRequest { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; private readonly BaseHttpClient _httpClient; private readonly ILucidaRateLimiter? _rateLimiter; public LucidaDownloadRequest(RemoteAlbum remoteAlbum, LucidaDownloadOptions? options) : base(remoteAlbum, options) { _httpClient = new BaseHttpClient(Options.BaseUrl, Options.RequestInterceptors, TimeSpan.FromSeconds(Options.RequestTimeout)); _rateLimiter = options?.RateLimiter; _requestContainer.Add(new OwnRequest(async (token) => { try { await ProcessDownloadAsync(token); return true; } catch (Exception ex) { LogAndAppendMessage($"Error processing download: {ex.Message}", LogLevel.Error); throw; } }, new RequestOptions() { CancellationToken = Token, DelayBetweenAttemps = Options.DelayBetweenAttemps, NumberOfAttempts = Options.NumberOfAttempts, Priority = RequestPriority.Low, Handler = Options.Handler })); } protected override async Task ProcessDownloadAsync(CancellationToken token) { _logger.Trace($"Processing {(Options.IsTrack ? "track" : "album")}: {ReleaseInfo.Title}"); if (Options.IsTrack) await ProcessSingleTrackAsync(Options.ItemId, token); else await ProcessAlbumAsync(Options.ItemId, token); } private async Task ProcessSingleTrackAsync(string downloadUrl, CancellationToken token) { LucidaTokens tokens = await LucidaTokenExtractor.ExtractTokensAsync(_httpClient, downloadUrl); if (!tokens.IsValid) throw new Exception("Failed to extract authentication tokens"); Track trackMetadata = CreateTrackFromLucidaData(new LucidaTrackModel { Title = ReleaseInfo.Title, TrackNumber = 1, Artist = _remoteAlbum.Artist?.Name ?? "Unknown Artist", DurationMs = 0 }, new LucidaAlbumModel { Title = ReleaseInfo.Album ?? ReleaseInfo.Title, Artist = _remoteAlbum.Artist?.Name ?? "Unknown Artist", ReleaseDate = ReleaseInfo.PublishDate.ToString("yyyy-MM-dd") }); Album albumMetadata = CreateAlbumFromLucidaData(new LucidaAlbumModel { Title = ReleaseInfo.Album ?? ReleaseInfo.Title, Artist = _remoteAlbum.Artist?.Name ?? "Unknown Artist", ReleaseDate = ReleaseInfo.PublishDate.ToString("yyyy-MM-dd"), TrackCount = 1 }); string fileName = BuildTrackFilename(trackMetadata, albumMetadata, AudioFormatHelper.GetFileExtensionForCodec(_remoteAlbum.Release.Codec.ToLower())); InitiateDownload(downloadUrl, tokens.Primary, tokens.Fallback, tokens.Expiry, fileName, token); _requestContainer.Add(_trackContainer); } private async Task ProcessAlbumAsync(string downloadUrl, CancellationToken token) { LucidaAlbumModel album = await LucidaMetadataExtractor.ExtractAlbumMetadataAsync(_httpClient, downloadUrl); _expectedTrackCount = album.Tracks.Count; _logger.Trace($"Found {album.Tracks.Count} tracks in album: {album.Title}"); for (int i = 0; i < album.Tracks.Count; i++) { LucidaTrackModel track = album.Tracks[i]; Track trackMetadata = CreateTrackFromLucidaData(track, album); Album albumMetadata = CreateAlbumFromLucidaData(album); string trackFileName = BuildTrackFilename(trackMetadata, albumMetadata, AudioFormatHelper.GetFileExtensionForCodec(_remoteAlbum.Release.Codec.ToLower())); try { string? trackUrl = !string.IsNullOrEmpty(track.Url) ? track.Url : track.OriginalServiceUrl; if (string.IsNullOrEmpty(trackUrl)) { _logger.Warn($"No URL available for track: {track.Title}"); continue; } InitiateDownload(trackUrl, album.PrimaryToken!, album.FallbackToken!, album.TokenExpiry, trackFileName, token); _logger.Trace($"Track {i + 1}/{album.Tracks.Count} completed: {track.Title}"); } catch (Exception ex) { LogAndAppendMessage($"Track {i + 1}/{album.Tracks.Count} failed: {track.Title} - {ex.Message}", LogLevel.Error); } } _requestContainer.Add(_trackContainer); } private void InitiateDownload(string url, string primaryToken, string fallbackToken, long expiry, string fileName, CancellationToken token) { OwnRequest downloadRequestWrapper = new(async (t) => { LucidaInitiationResult initiation; try { initiation = await InitiateDownloadWithRetryAsync( url, primaryToken, fallbackToken, expiry, t); _logger.Trace($"Initiation completed. Handoff: {initiation.HandoffId}, Server: {initiation.ServerName}"); } catch (Exception ex) { LogAndAppendMessage($"Initiation failed after retries: {ex.Message}", LogLevel.Error); return false; } try { if (!await PollForCompletionAsync(initiation.HandoffId, initiation.ServerName, t)) return false; } catch (Exception ex) { LogAndAppendMessage($"Polling failed: {ex.Message}", LogLevel.Error); return false; } try { string domain = ExtractDomainFromUrl(Options.BaseUrl); string downloadUrl = $"https://{initiation.ServerName}.{domain}/api/fetch/request/{initiation.HandoffId}/download"; LoadRequest downloadRequest = new(downloadUrl, new LoadRequestOptions() { CancellationToken = t, CreateSpeedReporter = true, SpeedReporterTimeout = 1000, Priority = RequestPriority.Normal, MaxBytesPerSecond = Options.MaxDownloadSpeed, DelayBetweenAttemps = Options.DelayBetweenAttemps, Filename = fileName, AutoStart = true, DestinationPath = _destinationPath.FullPath, Handler = Options.Handler, DeleteFilesOnFailure = true, RequestFailed = (_, __) => LogAndAppendMessage($"Download failed: {fileName}", LogLevel.Error), WriteMode = WriteMode.AppendOrTruncate, }); _trackContainer.Add(downloadRequest); return true; } catch (Exception ex) { LogAndAppendMessage($"Download request failed: {ex.Message}", LogLevel.Error); return false; } }, new RequestOptions() { AutoStart = true, Priority = RequestPriority.High, NumberOfAttempts = Options.NumberOfAttempts, DelayBetweenAttemps = Options.DelayBetweenAttemps, Handler = Options.Handler, CancellationToken = token }); _requestContainer.Add(downloadRequestWrapper); } private async Task InitiateDownloadWithRetryAsync(string url, string primaryToken, string fallbackToken, long expiry, CancellationToken token) { if (_rateLimiter == null) { _logger.Trace("No rate limiter configured, using direct request"); return await InitiateDownloadRequestAsync(url, primaryToken, fallbackToken, expiry, null, token); } int totalAttempts = 0; string? lastWorker = null; // Try different workers for (int workerRetry = 0; workerRetry < ILucidaRateLimiter.MAX_WORKER_RETRIES; workerRetry++) { token.ThrowIfCancellationRequested(); string worker = await _rateLimiter.WaitForAvailableWorkerAsync(token); totalAttempts++; lastWorker = worker; _logger.Trace($"Attempt {totalAttempts}: Using worker '{worker}' (retry {workerRetry + 1}/{ILucidaRateLimiter.MAX_WORKER_RETRIES})"); try { LucidaInitiationResult result = await InitiateDownloadRequestAsync(url, primaryToken, fallbackToken, expiry, worker, token); _rateLimiter.ReleaseWorker(worker); _logger.Debug($"Download initiated successfully on worker '{worker}'"); return result; } catch (LucidaRateLimitException) { _rateLimiter.MarkWorkerRateLimited(worker); _logger.Debug($"Worker '{worker}' rate limited, trying different worker"); } catch (Exception ex) when (ex is not OperationCanceledException) { _rateLimiter.ReleaseWorker(worker); throw; } } // All workers rate limited wait _logger.Debug($"All workers rate limited, waiting {ILucidaRateLimiter.WAIT_ON_ALL_BLOCKED_MS}ms before retry"); await Task.Delay(ILucidaRateLimiter.WAIT_ON_ALL_BLOCKED_MS, token); //Retry after waiting for (int waitRetry = 0; waitRetry < ILucidaRateLimiter.MAX_WAIT_RETRIES; waitRetry++) { token.ThrowIfCancellationRequested(); string worker = await _rateLimiter.WaitForAvailableWorkerAsync(token); totalAttempts++; lastWorker = worker; _logger.Trace($"Attempt {totalAttempts}: After wait, using worker '{worker}' (wait retry {waitRetry + 1}/{ILucidaRateLimiter.MAX_WAIT_RETRIES})"); try { LucidaInitiationResult result = await InitiateDownloadRequestAsync(url, primaryToken, fallbackToken, expiry, worker, token); _rateLimiter.ReleaseWorker(worker); _logger.Debug($"Download initiated successfully on worker '{worker}' after wait"); return result; } catch (LucidaRateLimitException) { _rateLimiter.MarkWorkerRateLimited(worker); _logger.Debug($"Worker '{worker}' still rate limited after wait"); } catch (Exception ex) when (ex is not OperationCanceledException) { _rateLimiter.ReleaseWorker(worker); throw; } } throw new Exception($"Failed to initiate download after {totalAttempts} attempts across all workers (last worker: {lastWorker})"); } private async Task InitiateDownloadRequestAsync( string url, string primaryToken, string fallbackToken, long expiry, string? forcedWorker, CancellationToken token) { try { LucidaDownloadRequestInfo request = LucidaDownloadRequestInfo.CreateWithTokens(url, primaryToken, fallbackToken, expiry); string requestBody = JsonSerializer.Serialize(request, _jsonOptions); string apiEndpoint = "%2Fapi%2Ffetch%2Fstream%2Fv2"; if (!string.IsNullOrEmpty(forcedWorker)) apiEndpoint += $"&force={forcedWorker}"; string requestUrl = $"{_httpClient.BaseUrl}/api/load?url={apiEndpoint}"; _logger.Trace($"Initiating download from URL: {url}, Forced worker: {forcedWorker ?? "auto"}"); HttpRequestMessage httpRequest = new(HttpMethod.Post, requestUrl); httpRequest.Headers.Add("Origin", _httpClient.BaseUrl); httpRequest.Headers.Add("Referer", $"{_httpClient.BaseUrl}/?url={Uri.EscapeDataString(url)}"); httpRequest.Content = new StringContent(requestBody, Encoding.UTF8, "text/plain"); HttpResponseMessage response = await _httpClient.PostAsync(httpRequest); string responseContent = await response.Content.ReadAsStringAsync(token); response.EnsureSuccessStatusCode(); if (_rateLimiter?.IsRateLimitedResponse(responseContent) == true) throw new LucidaRateLimitException($"Rate limited on worker '{forcedWorker}'", forcedWorker); LucidaDownloadResponse? downloadResponse = JsonSerializer.Deserialize(responseContent, _jsonOptions); if (downloadResponse?.Success == true && !string.IsNullOrEmpty(downloadResponse.Handoff)) { string serverName = downloadResponse.Server ?? downloadResponse.Name ?? forcedWorker ?? "hund"; _rateLimiter?.EnsureWorkerRegistered(serverName); return new LucidaInitiationResult { HandoffId = downloadResponse.Handoff, ServerName = serverName }; } string errorInfo = downloadResponse != null ? $"Success: {downloadResponse.Success}, Handoff: {downloadResponse.Handoff}, Server: {downloadResponse.Server}, Name: {downloadResponse.Name}, Error: {downloadResponse.Error}" : "Failed to deserialize response"; throw new Exception($"Failed to initiate download: {errorInfo}"); } catch (LucidaRateLimitException) { throw; } catch (Exception ex) { LogAndAppendMessage($"Error initiating download: {ex.Message}", LogLevel.Error); throw; } } private async Task PollForCompletionAsync(string handoffId, string serverName, CancellationToken token) { const int baseAttempts = 15; int delayMs = 5000; int serviceUnavailableExtensions = 1; const int maxServiceUnavailableExtensions = 20; await Task.Delay(delayMs * 2, token); int totalAttempts = baseAttempts + (serviceUnavailableExtensions * 5); for (int attempt = 1; attempt <= totalAttempts; attempt++) { if (token.IsCancellationRequested) return false; bool acquiredSlot = false; try { if (_rateLimiter is not null) { await _rateLimiter.AcquirePollingSlotAsync(serverName, token); acquiredSlot = true; } string statusUrl = $"https://{serverName}.{ExtractDomainFromUrl(Options.BaseUrl)}/api/fetch/request/{handoffId}"; string responseContent = await _httpClient.GetStringAsync(statusUrl, token); if (_rateLimiter?.IsRateLimitedResponse(responseContent) == true) { _logger.Debug($"Polling rate limited on worker '{serverName}', treating as transient"); } else { LucidaStatusResponse? status = JsonSerializer.Deserialize(responseContent, _jsonOptions); if (status?.Success == true && status.Status == "completed") return true; if (!string.IsNullOrEmpty(status?.Error) && status.Error != "Request not found." && status.Error != "No such request") throw new Exception($"Server error: {status.Error}"); if (!string.IsNullOrEmpty(status?.Status)) _logger.Trace($"Poll {attempt}: status={status.Status}"); } } catch (HttpRequestException httpEx) when (httpEx.StatusCode == System.Net.HttpStatusCode.InternalServerError) { _logger.Error($"Polling failed with 500 Internal Server Error. Handoff ID may be invalid: {httpEx.Message}"); throw new Exception($"Server internal error. Handoff ID invalid: {httpEx.Message}"); } catch (HttpRequestException httpEx) when (httpEx.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable) { if (attempt >= baseAttempts && serviceUnavailableExtensions < maxServiceUnavailableExtensions) { serviceUnavailableExtensions++; totalAttempts = baseAttempts + (serviceUnavailableExtensions * 5); _logger.Warn($"Service unavailable. Extending polling with 5-minute intervals (extension {serviceUnavailableExtensions}/{maxServiceUnavailableExtensions})"); await Task.Delay(TimeSpan.FromMinutes(5), token); continue; } } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.Trace($"Polling attempt {attempt} failed: {ex.Message}"); } finally { if (acquiredSlot) _rateLimiter?.ReleasePollingSlot(serverName); } await Task.Delay(delayMs, token); delayMs = Math.Min(delayMs * 2, 6000); } throw new Exception("Download did not complete within expected time"); } private static string ExtractDomainFromUrl(string url) { if (string.IsNullOrEmpty(url)) return "lucida.to"; return url.Replace("https://", "").Replace("http://", "").TrimEnd('/'); } private Album CreateAlbumFromLucidaData(LucidaAlbumModel? albumInfo) { DateTime releaseDate = ReleaseInfo.PublishDate; if (!string.IsNullOrEmpty(albumInfo?.ReleaseDate) && DateTime.TryParse(albumInfo.ReleaseDate, out DateTime parsedDate)) releaseDate = parsedDate; return new Album { Title = albumInfo?.Title ?? ReleaseInfo.Album, ReleaseDate = releaseDate, Artist = new NzbDrone.Core.Datastore.LazyLoaded(new Artist { Name = albumInfo?.Artist ?? ReleaseInfo.Artist, }), AlbumReleases = new NzbDrone.Core.Datastore.LazyLoaded>( [ new() { TrackCount = albumInfo?.TrackCount ?? 0, Title = albumInfo?.Title ?? ReleaseInfo.Album, Duration = (int)(albumInfo?.GetTotalDurationMs() ?? 0) } ]), Genres = _remoteAlbum.Albums?.FirstOrDefault()?.Genres, }; } private Track CreateTrackFromLucidaData(LucidaTrackModel trackInfo, LucidaAlbumModel? albumInfo) => new() { Title = trackInfo.Title, TrackNumber = trackInfo.TrackNumber.ToString(), AbsoluteTrackNumber = trackInfo.TrackNumber, Duration = (int)trackInfo.DurationMs, Explicit = trackInfo.IsExplicit, Artist = new NzbDrone.Core.Datastore.LazyLoaded(new Artist { Name = trackInfo.Artist ?? albumInfo?.Artist ?? ReleaseInfo.Artist ?? _remoteAlbum.Artist?.Name, }), MediumNumber = trackInfo.DiscNumber }; } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaInitiationResult.cs ================================================ namespace Tubifarry.Download.Clients.Lucida; public sealed record LucidaInitiationResult { public required string HandoffId { get; init; } public required string ServerName { get; init; } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaMetadataExtractor.cs ================================================ using Jint; using Jint.Native; using NLog; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.Json; using Tubifarry.Download.Base; using Tubifarry.Indexers.Lucida; namespace Tubifarry.Download.Clients.Lucida; /// /// Metadata extractor for Lucida pages /// Uses System.Text.Json exclusively with Jint fallback for JavaScript execution /// public static partial class LucidaMetadataExtractor { private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(LucidaMetadataExtractor)); private static readonly JsonSerializerOptions Json = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true }; public static async Task ExtractAlbumMetadataAsync(BaseHttpClient httpClient, string url) { LucidaAlbumModel? album = null; try { album = await ExtractViaApiAsync(httpClient, url); if (album is { IsValid: true }) _logger.Debug($"Native API returned valid album: {album.Title} ({album.Tracks.Count} tracks)"); } catch (Exception ex) { _logger.Debug($"Native API metadata failed, falling back to HTML: {ex.Message}"); } if (album is not { IsValid: true }) { try { album = await ExtractViaHtmlAsync(httpClient, url); if (album is { IsValid: true }) _logger.Debug($"HTML extraction returned valid album: {album.Title} ({album.Tracks.Count} tracks)"); } catch (Exception ex) { _logger.Error(ex, "HTML metadata extraction also failed for {0}", url); } } if (album is not { IsValid: true }) { _logger.Error("All metadata extraction strategies failed for {0}", url); return new LucidaAlbumModel { OriginalServiceUrl = url }; } bool hasCsrfTokens = album.Tracks.Any(t => t.HasValidTokens) || album.HasValidTokens; if (!hasCsrfTokens) { _logger.Debug("Album lacks CSRF tokens, fetching from HTML page..."); try { await EnrichWithCsrfTokensAsync(httpClient, url, album); } catch (Exception ex) { _logger.Warn(ex, "Failed to enrich album with CSRF tokens"); } } album.OriginalServiceUrl = url; return album; } private static async Task ExtractViaApiAsync(BaseHttpClient httpClient, string serviceUrl) { string apiUrl = $"{httpClient.BaseUrl}/api/load?url={Uri.EscapeDataString($"/api/fetch/metadata?url={serviceUrl}")}"; _logger.Trace($"Fetching metadata via API: {apiUrl}"); using HttpResponseMessage response = await httpClient.GetAsync(apiUrl); string body = await response.Content.ReadAsStringAsync(); if (response.StatusCode == HttpStatusCode.NotFound || body.Contains("404 Not Found", StringComparison.OrdinalIgnoreCase) || IsCharIndexed404(body)) { _logger.Debug("Metadata API returned 404"); return null; } if (!response.IsSuccessStatusCode) { _logger.Debug($"Metadata API returned HTTP {(int)response.StatusCode}"); return null; } string? contentType = response.Content.Headers.ContentType?.MediaType; if (string.Equals(contentType, "text/html", StringComparison.OrdinalIgnoreCase)) { _logger.Debug("Metadata API returned HTML instead of JSON"); return null; } LucidaInfo? info; try { info = JsonSerializer.Deserialize(body, Json); } catch (JsonException ex) { _logger.Debug($"Metadata API response not valid JSON: {ex.Message}"); return null; } if (info is null || !info.Success) { _logger.Debug($"Metadata API returned success={info?.Success}, type={info?.Type}"); return null; } if (string.Equals(info.Type, "album", StringComparison.OrdinalIgnoreCase)) return ConvertToAlbum(info); if (string.Equals(info.Type, "track", StringComparison.OrdinalIgnoreCase)) return ConvertSingleTrackToAlbum(info); _logger.Debug($"Metadata API returned unexpected type: {info.Type}"); return null; } private static async Task ExtractViaHtmlAsync(BaseHttpClient httpClient, string serviceUrl) { string pageUrl = $"{httpClient.BaseUrl}/?url={Uri.EscapeDataString(serviceUrl)}"; _logger.Trace($"Fetching HTML page: {pageUrl}"); string html; try { using HttpResponseMessage response = await httpClient.GetAsync(pageUrl); html = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { _logger.Debug($"HTML page returned HTTP {(int)response.StatusCode}"); return null; } } catch (Exception ex) { _logger.Debug($"HTML page fetch failed: {ex.Message}"); return null; } if (string.IsNullOrWhiteSpace(html) || html.Length < 200) { _logger.Debug("HTML page is empty or too short"); return null; } LucidaInfo? info = ExtractInfoFromHtml(html); if (info is null || !info.Success) { _logger.Warn($"HTML/Jint extraction failed: success={info?.Success}, type={info?.Type}"); return null; } LucidaAlbumModel? album = info.Type?.ToLowerInvariant() switch { "album" => ConvertToAlbum(info), "track" => ConvertSingleTrackToAlbum(info), _ => null }; if (album is null) { _logger.Warn($"Unsupported info type from HTML: {info.Type}"); return null; } LucidaTokens pageTokens = LucidaTokenExtractor.ExtractTokensFromHtml(html); if (pageTokens.IsValid) ApplyTokensToAlbum(album, pageTokens); album.DetailPageUrl = pageUrl; return album; } private static async Task EnrichWithCsrfTokensAsync( BaseHttpClient httpClient, string serviceUrl, LucidaAlbumModel album) { string pageUrl = $"{httpClient.BaseUrl}/?url={Uri.EscapeDataString(serviceUrl)}"; string html; try { using HttpResponseMessage resp = await httpClient.GetAsync(pageUrl); html = await resp.Content.ReadAsStringAsync(); } catch (Exception ex) { _logger.Debug($"CSRF token fetch failed: {ex.Message}"); return; } LucidaTokens pageTokens = LucidaTokenExtractor.ExtractTokensFromHtml(html); if (pageTokens.IsValid) { ApplyTokensToAlbum(album, pageTokens); _logger.Debug("Applied page-level CSRF tokens to album"); return; } LucidaInfo? info = ExtractInfoFromHtml(html); if (info?.Tracks is null) return; foreach (LucidaTrackInfo trackInfo in info.Tracks) { if (string.IsNullOrEmpty(trackInfo.Csrf)) continue; LucidaTrackModel? match = album.Tracks.FirstOrDefault(t => (!string.IsNullOrEmpty(t.Id) && t.Id == trackInfo.Id) || (!string.IsNullOrEmpty(t.Url) && t.Url == trackInfo.Url) || (t.TrackNumber == trackInfo.TrackNumber && t.DiscNumber == trackInfo.DiscNumber)); if (match is not null) { match.PrimaryToken = trackInfo.Csrf; match.FallbackToken = trackInfo.CsrfFallback ?? trackInfo.Csrf; match.TokenExpiry = pageTokens.Expiry > 0 ? pageTokens.Expiry : DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeSeconds(); } } _logger.Debug($"Applied per-track CSRF tokens: {album.Tracks.Count(t => t.HasValidTokens)}/{album.Tracks.Count} tracks"); } private static LucidaInfo? ExtractInfoFromHtml(string html) { try { string jsonArray = ExtractJsonArrayFromHtml(html); if (string.IsNullOrEmpty(jsonArray)) { _logger.Debug("No data JSON array found in HTML"); return null; } _logger.Trace($"Extracted data array: {jsonArray.Length} chars"); return ParseDataArray(jsonArray); } catch (Exception ex) { _logger.Debug(ex, "HTML info extraction failed"); return null; } } private static string ExtractJsonArrayFromHtml(string html) { if (string.IsNullOrEmpty(html)) return string.Empty; int dataIdx = -1; string[] needles = ["const data = [", "var data = [", "data\t=\t[", "data = ["]; foreach (string needle in needles) { dataIdx = html.IndexOf(needle, StringComparison.Ordinal); if (dataIdx >= 0) { dataIdx = html.IndexOf('[', dataIdx); break; } } if (dataIdx < 0) { _logger.Debug("No 'data = [' assignment found in HTML"); return string.Empty; } int depth = 0; bool inString = false; bool escaped = false; for (int i = dataIdx; i < html.Length; i++) { char c = html[i]; if (escaped) { escaped = false; continue; } if (c == '\\' && inString) { escaped = true; continue; } if (c == '"' && !escaped) { inString = !inString; continue; } if (inString) continue; if (c == '[') depth++; else if (c == ']') depth--; if (depth == 0) { string jsonArray = html[dataIdx..(i + 1)]; _logger.Trace($"Extracted JSON array: {jsonArray.Length} chars"); return jsonArray; } } _logger.Debug("Bracket matching failed: unbalanced brackets in data array"); return string.Empty; } private static LucidaInfo? ParseDataArray(string jsonArray) { if (string.IsNullOrEmpty(jsonArray)) return null; try { using JsonDocument doc = JsonDocument.Parse(jsonArray); foreach (JsonElement entry in doc.RootElement.EnumerateArray()) { if (entry.TryGetProperty("type", out JsonElement typeEl) && typeEl.GetString() == "data" && entry.TryGetProperty("data", out JsonElement dataEl) && dataEl.TryGetProperty("info", out JsonElement infoEl)) { string infoJson = infoEl.GetRawText(); _logger.Trace($"Found info object via direct JSON: {infoJson.Length} chars"); return JsonSerializer.Deserialize(infoJson, Json); } } _logger.Debug("No 'data.info' entry found in JSON array"); return null; } catch (JsonException ex) { _logger.Debug($"Direct JSON parse failed, trying Jint fallback: {ex.Message}"); } return ExecuteJintOnDataArray(jsonArray); } private static LucidaInfo? ExecuteJintOnDataArray(string jsonArray) { try { Engine engine = new(opts => opts .TimeoutInterval(TimeSpan.FromSeconds(5)) .LimitMemory(50_000_000)); engine.Execute($@" var __data = {jsonArray}; var __infoJson = null; if (__data && __data.length) {{ for (var i = 0; i < __data.length; i++) {{ var e = __data[i]; if (e && e.type === 'data' && e.data && e.data.info) {{ __infoJson = JSON.stringify(e.data.info); break; }} }} }} "); JsValue val = engine.GetValue("__infoJson"); if (val.IsNull() || val.IsUndefined()) { _logger.Debug("Jint fallback: no info found in data array"); return null; } string json = val.AsString(); _logger.Trace($"Jint fallback JSON.stringify: {json.Length} chars"); return JsonSerializer.Deserialize(json, Json); } catch (Exception ex) { _logger.Error(ex, "Jint fallback extraction failed"); return null; } } private static LucidaAlbumModel ConvertToAlbum(LucidaInfo info) { _logger.Trace($"Converting info to album: title={info.Title}, " + $"trackCount={info.TrackCount}, tracks.length={info.Tracks?.Length ?? 0}"); LucidaAlbumModel album = new() { Id = info.Id, Title = info.Title, Artist = info.Artists?.FirstOrDefault()?.Name ?? string.Empty, TrackCount = info.TrackCount, DiscCount = info.DiscCount, Upc = info.Upc, Copyright = info.Copyright, ReleaseDate = info.ReleaseDate, ServiceName = info.Stats?.Service }; if (info.Artists is not null) album.Artists.AddRange(info.Artists); if (info.CoverArtwork is not null) album.CoverArtworks.AddRange(info.CoverArtwork); album.CoverUrl = album.GetBestCoverArtUrl(); if (info.Tracks is not null) { foreach (LucidaTrackInfo ti in info.Tracks) { LucidaTrackModel track = new() { Id = ti.Id, Title = ti.Title, Artist = ti.Artists?.FirstOrDefault()?.Name ?? album.Artist, DurationMs = ti.DurationMs, TrackNumber = ti.TrackNumber, DiscNumber = ti.DiscNumber, IsExplicit = ti.Explicit, Isrc = ti.Isrc, Copyright = ti.Copyright, Url = ti.Url, PrimaryToken = ti.Csrf, FallbackToken = ti.CsrfFallback }; if (ti.Artists is not null) track.Artists.AddRange(ti.Artists.Select( a => new LucidaArtist(a.Id, a.Name, a.Url, a.Pictures?.ToList()))); if (ti.Composers is not null) track.Composers.AddRange(ti.Composers); if (ti.Producers is not null) track.Producers.AddRange(ti.Producers); if (ti.Lyricists is not null) track.Lyricists.AddRange(ti.Lyricists); album.Tracks.Add(track); } } if (!string.IsNullOrEmpty(info.ReleaseDate) && info.ReleaseDate.Length >= 4) album.Year = info.ReleaseDate[..4]; _logger.Trace($"Album conversion done: {album.Tracks.Count} tracks"); return album; } private static LucidaAlbumModel ConvertSingleTrackToAlbum(LucidaInfo info) { _logger.Trace($"Converting single track to album wrapper: {info.Title}"); string albumTitle = info.Album?.Title ?? info.Title; string albumArtist = info.Artists?.FirstOrDefault()?.Name ?? string.Empty; LucidaAlbumModel album = new() { Id = info.Album?.Id ?? info.Id, Title = albumTitle, Artist = albumArtist, TrackCount = 1, DiscCount = 1, Copyright = info.Copyright, ReleaseDate = info.Album?.ReleaseDate ?? info.ReleaseDate, ServiceName = info.Stats?.Service }; if (info.Artists is not null) album.Artists.AddRange(info.Artists); LucidaArtworkInfo[]? artworks = info.Album?.CoverArtwork ?? info.CoverArtwork; if (artworks is not null) album.CoverArtworks.AddRange(artworks); album.CoverUrl = album.GetBestCoverArtUrl(); LucidaTrackModel track = new() { Id = info.Id, Title = info.Title, Artist = albumArtist, DurationMs = info.DurationMs, TrackNumber = info.TrackNumber, DiscNumber = info.DiscNumber, IsExplicit = info.Explicit, Isrc = info.Isrc, Copyright = info.Copyright, Url = info.Url }; if (info.Artists is not null) track.Artists.AddRange(info.Artists.Select( a => new LucidaArtist(a.Id, a.Name, a.Url, a.Pictures?.ToList()))); if (info.Composers is not null) track.Composers.AddRange(info.Composers); if (info.Producers is not null) track.Producers.AddRange(info.Producers); if (info.Lyricists is not null) track.Lyricists.AddRange(info.Lyricists); album.Tracks.Add(track); if (!string.IsNullOrEmpty(album.ReleaseDate) && album.ReleaseDate.Length >= 4) album.Year = album.ReleaseDate[..4]; return album; } private static void ApplyTokensToAlbum(LucidaAlbumModel album, LucidaTokens tokens) { album.PrimaryToken = tokens.Primary; album.FallbackToken = tokens.Fallback; album.TokenExpiry = tokens.Expiry; foreach (LucidaTrackModel track in album.Tracks) { if (string.IsNullOrEmpty(track.PrimaryToken)) { track.PrimaryToken = tokens.Primary; track.FallbackToken = tokens.Fallback; track.TokenExpiry = tokens.Expiry; } } } private static bool IsCharIndexed404(string body) { if (string.IsNullOrEmpty(body) || body.Length < 30) return false; return body.StartsWith("{\"0\":\"<", StringComparison.Ordinal) || (body.StartsWith("{\"0\":\"", StringComparison.Ordinal) && body.Contains("\"fromExternal\"", StringComparison.Ordinal)); } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; namespace Tubifarry.Download.Clients.Lucida { public class LucidaProviderSettingsValidator : AbstractValidator { public LucidaProviderSettingsValidator() { // Validate BaseUrl RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); // Validate DownloadPath RuleFor(x => x.DownloadPath) .IsValidPath() .WithMessage("Download path must be a valid directory."); // Validate RequestTimeout RuleFor(x => x.RequestTimeout) .GreaterThanOrEqualTo(5) .LessThanOrEqualTo(300) .WithMessage("Request timeout must be between 5 and 300 seconds."); // Validate ConnectionRetries RuleFor(x => x.ConnectionRetries) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(10) .WithMessage("Connection retries must be between 1 and 10."); // Validate MaxParallelDownloads RuleFor(x => x.MaxParallelDownloads) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(10) .WithMessage("Max parallel downloads must be between 1 and 10."); // Validate MaxDownloadSpeed RuleFor(x => x.MaxDownloadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Max download speed must be greater than or equal to 0.") .LessThanOrEqualTo(100_000) .WithMessage("Max download speed must be less than or equal to 100 MB/s."); } } /// /// Configuration settings for the Lucida download client /// public class LucidaProviderSettings : IProviderConfig { private static readonly LucidaProviderSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Directory where downloaded files will be saved")] public string DownloadPath { get; set; } = string.Empty; [FieldDefinition(1, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the Lucida instance", Placeholder = "https://lucida.to")] public string BaseUrl { get; set; } = "https://lucida.to"; [FieldDefinition(2, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for HTTP requests to Lucida", Advanced = true)] public int RequestTimeout { get; set; } = 30; [FieldDefinition(3, Type = FieldType.Number, Label = "Connection Retries", HelpText = "Number of times to retry failed connections", Advanced = true)] public int ConnectionRetries { get; set; } = 3; [FieldDefinition(4, Type = FieldType.Number, Label = "Max Parallel Downloads", HelpText = "Maximum number of downloads that can run simultaneously")] public int MaxParallelDownloads { get; set; } = 2; [FieldDefinition(5, Label = "Max Download Speed", Type = FieldType.Number, HelpText = "Set to 0 for unlimited speed. Limits download speed per file.", Unit = "KB/s", Advanced = true)] public int MaxDownloadSpeed { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaRateLimitException.cs ================================================ namespace Tubifarry.Download.Clients.Lucida { public class LucidaRateLimitException : Exception { public string? WorkerName { get; } public LucidaRateLimitException(string message, string? workerName = null) : base(message) => WorkerName = workerName; public LucidaRateLimitException(string message, Exception innerException, string? workerName = null) : base(message, innerException) => WorkerName = workerName; } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaRateLimiter.cs ================================================ using NLog; using System.Collections.Concurrent; using System.Text; using System.Text.Json; namespace Tubifarry.Download.Clients.Lucida; public sealed class LucidaRateLimiter : ILucidaRateLimiter { private readonly Logger _logger; private readonly ConcurrentDictionary _slots = new(StringComparer.OrdinalIgnoreCase); private readonly object _globalTimingLock = new(); private DateTime _lastRequestTime = DateTime.MinValue; public IReadOnlyCollection Workers => [.. _slots.Keys]; public LucidaRateLimiter(Logger logger) { _logger = logger; EnsureWorkerRegistered(ILucidaRateLimiter.WORKER_MAUS); EnsureWorkerRegistered(ILucidaRateLimiter.WORKER_HUND); EnsureWorkerRegistered(ILucidaRateLimiter.WORKER_KATZE); } public void EnsureWorkerRegistered(string workerName) { if (string.IsNullOrEmpty(workerName)) return; _slots.GetOrAdd(workerName, static name => new WorkerSlot(name)); } public async Task WaitForAvailableWorkerAsync(CancellationToken cancellationToken) { int attempts = 0; while (!cancellationToken.IsCancellationRequested) { attempts++; string? best = PickBestWorker(); if (best is not null) { WorkerSlot slot = _slots[best]; bool acquired = await slot.RequestSemaphore.WaitAsync( TimeSpan.FromSeconds(5), cancellationToken); if (acquired) { await EnforceGlobalDelayAsync(cancellationToken); Interlocked.Increment(ref slot.ActiveRequests); _logger.Trace($"Acquired worker '{best}' (attempt {attempts})"); return best; } } await Task.Delay(1000, cancellationToken); } throw new OperationCanceledException("Cancelled while waiting for an available worker"); } public void MarkWorkerRateLimited(string workerName) { if (string.IsNullOrEmpty(workerName)) return; EnsureWorkerRegistered(workerName); WorkerSlot slot = _slots[workerName]; DateTime until = DateTime.UtcNow.AddSeconds(ILucidaRateLimiter.COOLDOWN_SECONDS); slot.RateLimitedUntil = until; _logger.Debug($"Worker '{workerName}' rate-limited until {until:HH:mm:ss}"); } public void ReleaseWorker(string workerName) { if (string.IsNullOrEmpty(workerName) || !_slots.TryGetValue(workerName, out WorkerSlot? slot)) return; try { slot.RequestSemaphore.Release(); } catch (SemaphoreFullException) { _logger.Trace($"Semaphore already at max for worker '{workerName}'"); } int current = Interlocked.Decrement(ref slot.ActiveRequests); if (current < 0) Interlocked.Exchange(ref slot.ActiveRequests, 0); slot.LastSuccessfulRequest = DateTime.UtcNow; _logger.Trace($"Released worker '{workerName}'"); } public bool IsRateLimitedResponse(string responseContent) { if (string.IsNullOrEmpty(responseContent)) return false; if (!responseContent.StartsWith("{\"0\":", StringComparison.Ordinal)) return false; try { string decoded = DecodeJsonEncodedHtml(responseContent); return decoded.Contains("404", StringComparison.OrdinalIgnoreCase) && decoded.Contains("Not Found", StringComparison.OrdinalIgnoreCase); } catch { return false; } } public IReadOnlyDictionary GetWorkerStates() { Dictionary snapshot = new(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair pair in _slots) { WorkerSlot slot = pair.Value; bool onCooldown = slot.RateLimitedUntil.HasValue && slot.RateLimitedUntil.Value > DateTime.UtcNow; snapshot[pair.Key] = new LucidaWorkerState { Name = pair.Key, IsAvailable = !onCooldown, RateLimitedUntil = slot.RateLimitedUntil, ActiveRequests = Interlocked.CompareExchange(ref slot.ActiveRequests, 0, 0), ActivePolling = Interlocked.CompareExchange(ref slot.ActivePolling, 0, 0), LastSuccessfulRequest = slot.LastSuccessfulRequest }; } return snapshot; } public async Task AcquirePollingSlotAsync(string workerName, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(workerName)) return; EnsureWorkerRegistered(workerName); WorkerSlot slot = _slots[workerName]; await slot.PollingSemaphore.WaitAsync(cancellationToken); Interlocked.Increment(ref slot.ActivePolling); } public void ReleasePollingSlot(string workerName) { if (string.IsNullOrEmpty(workerName) || !_slots.TryGetValue(workerName, out WorkerSlot? slot)) return; try { slot.PollingSemaphore.Release(); } catch (SemaphoreFullException) { } int current = Interlocked.Decrement(ref slot.ActivePolling); if (current < 0) Interlocked.Exchange(ref slot.ActivePolling, 0); } private string? PickBestWorker() { string? best = null; int bestScore = int.MaxValue; foreach (KeyValuePair pair in _slots) { WorkerSlot slot = pair.Value; if (slot.RateLimitedUntil.HasValue && slot.RateLimitedUntil.Value > DateTime.UtcNow) continue; int active = Interlocked.CompareExchange(ref slot.ActiveRequests, 0, 0); if (active >= ILucidaRateLimiter.MAX_CONCURRENT_PER_WORKER) continue; if (active < bestScore) { bestScore = active; best = pair.Key; } } if (best is null) { foreach (KeyValuePair pair in _slots) { WorkerSlot slot = pair.Value; bool cooldownExpired = !slot.RateLimitedUntil.HasValue || slot.RateLimitedUntil.Value <= DateTime.UtcNow; if (cooldownExpired) return pair.Key; } } return best; } private async Task EnforceGlobalDelayAsync(CancellationToken cancellationToken) { TimeSpan delay; lock (_globalTimingLock) { TimeSpan elapsed = DateTime.UtcNow - _lastRequestTime; delay = TimeSpan.FromMilliseconds(ILucidaRateLimiter.MIN_REQUEST_DELAY_MS) - elapsed; _lastRequestTime = DateTime.UtcNow; } if (delay > TimeSpan.Zero) { _logger.Trace($"Enforcing global request delay: {delay.TotalMilliseconds:F0}ms"); await Task.Delay(delay, cancellationToken); lock (_globalTimingLock) { _lastRequestTime = DateTime.UtcNow; } } } private static string DecodeJsonEncodedHtml(string jsonEncoded) { Dictionary? data = JsonSerializer.Deserialize>(jsonEncoded); if (data is null) return string.Empty; StringBuilder sb = new(); for (int i = 0; ; i++) { if (data.TryGetValue(i.ToString(), out JsonElement element) && element.ValueKind == JsonValueKind.String) sb.Append(element.GetString()); else break; } return sb.ToString(); } private sealed class WorkerSlot { public readonly SemaphoreSlim RequestSemaphore; public readonly SemaphoreSlim PollingSemaphore; public int ActiveRequests; public int ActivePolling; public DateTime? RateLimitedUntil; public DateTime? LastSuccessfulRequest; public WorkerSlot(string name) { _ = name; RequestSemaphore = new SemaphoreSlim( ILucidaRateLimiter.MAX_CONCURRENT_PER_WORKER, ILucidaRateLimiter.MAX_CONCURRENT_PER_WORKER); PollingSemaphore = new SemaphoreSlim( ILucidaRateLimiter.POLLING_MAX_CONCURRENT, ILucidaRateLimiter.POLLING_MAX_CONCURRENT); } } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaTokenExtractor.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using System.Text; using System.Text.RegularExpressions; using Tubifarry.Download.Base; using Tubifarry.Indexers.Lucida; namespace Tubifarry.Download.Clients.Lucida { public static partial class LucidaTokenExtractor { private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(LucidaTokenExtractor)); public static async Task ExtractTokensAsync(BaseHttpClient httpClient, string url) { try { string lucidaUrl = $"{httpClient.BaseUrl}/?url={Uri.EscapeDataString(url)}"; string html = await httpClient.GetStringAsync(lucidaUrl); return ExtractTokensFromHtml(html); } catch (Exception ex) { _logger.Error(ex, "Token extraction failed for {0}", url); return LucidaTokens.Empty; } } public static LucidaTokens ExtractTokensFromHtml(string html) { if (string.IsNullOrEmpty(html)) return LucidaTokens.Empty; try { Match match = TrackTokenRegex().Match(html); if (!match.Success) { _logger.Debug("No track token pattern found"); return LucidaTokens.Empty; } string encodedToken = match.Groups[1].Value; long expiry = long.TryParse(match.Groups[2].Value, out long e) ? e : DateTimeOffset.UtcNow.AddDays(30).ToUnixTimeSeconds(); if (encodedToken == "album") { _logger.Debug("Album page detected, tokens are per-track csrf values in metadata"); return LucidaTokens.Empty; } string decoded = DoubleBase64Decode(encodedToken); if (string.IsNullOrEmpty(decoded)) return LucidaTokens.Empty; return new LucidaTokens(decoded, decoded, expiry); } catch (Exception ex) { _logger.Error(ex, "Error extracting tokens from HTML"); return LucidaTokens.Empty; } } private static string DoubleBase64Decode(string encoded) { if (string.IsNullOrEmpty(encoded)) return string.Empty; try { string normalized = NormalizeBase64(encoded); byte[] firstBytes = Convert.FromBase64String(normalized); string firstDecode = Encoding.UTF8.GetString(firstBytes); string firstNormalized = NormalizeBase64(firstDecode); byte[] secondBytes = Convert.FromBase64String(firstNormalized); return Encoding.UTF8.GetString(secondBytes); } catch (Exception ex) { _logger.Debug(ex, "Failed to double-decode token"); return string.Empty; } } private static string NormalizeBase64(string input) { if (string.IsNullOrEmpty(input)) return input; string normalized = input.Replace('-', '+').Replace('_', '/'); int paddingNeeded = (4 - normalized.Length % 4) % 4; if (paddingNeeded > 0) normalized += new string('=', paddingNeeded); return normalized; } [GeneratedRegex(@"\b""?token""?\s*:\s*""([A-Za-z0-9+/=_-]{16,})""\s*,\s*\btokenExpiry\s*:\s*(\d+)", RegexOptions.Compiled)] private static partial Regex TrackTokenRegex(); } } ================================================ FILE: Tubifarry/Download/Clients/Lucida/LucidaWorkerState.cs ================================================ namespace Tubifarry.Download.Clients.Lucida; public sealed record LucidaWorkerState { public string Name { get; init; } = string.Empty; public bool IsAvailable { get; init; } public DateTime? RateLimitedUntil { get; init; } public int ActiveRequests { get; init; } public int ActivePolling { get; init; } public DateTime? LastSuccessfulRequest { get; init; } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/ISlskdApiClient.cs ================================================ using FluentValidation.Results; using Tubifarry.Download.Clients.Soulseek.Models; namespace Tubifarry.Download.Clients.Soulseek; public class SlskdUserTransfers { public string Username { get; set; } = string.Empty; public IEnumerable Directories { get; set; } = []; } public class SlskdEventRecord { public Guid Id { get; set; } public DateTime Timestamp { get; set; } public string Type { get; set; } = string.Empty; public string Data { get; set; } = string.Empty; } public interface ISlskdApiClient { Task<(List Enqueued, List Failed)> EnqueueDownloadAsync(SlskdProviderSettings settings, string username, IEnumerable<(string Filename, long Size)> files); Task> GetAllTransfersAsync(SlskdProviderSettings settings, bool includeRemoved = false); Task GetUserTransfersAsync(SlskdProviderSettings settings, string username); Task GetTransferAsync(SlskdProviderSettings settings, string username, string fileId); Task GetQueuePositionAsync(SlskdProviderSettings settings, string username, string fileId); Task DeleteTransferAsync(SlskdProviderSettings settings, string username, string fileId, bool remove = false); Task DeleteAllCompletedAsync(SlskdProviderSettings settings); Task GetDownloadPathAsync(SlskdProviderSettings settings); Task TestConnectionAsync(SlskdProviderSettings settings); Task<(List Events, int TotalCount)> GetEventsAsync(SlskdProviderSettings settings, int offset, int limit); } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/ISlskdDownloadManager.cs ================================================ using NzbDrone.Common.Disk; using NzbDrone.Core.Download; using NzbDrone.Core.Parser.Model; namespace Tubifarry.Download.Clients.Soulseek; public interface ISlskdDownloadManager { Task DownloadAsync(RemoteAlbum remoteAlbum, int definitionId, SlskdProviderSettings settings); IEnumerable GetItems(int definitionId, SlskdProviderSettings settings, OsPath remotePath); void RemoveItem(DownloadClientItem clientItem, bool deleteData, int definitionId, SlskdProviderSettings settings); } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/Models/DownloadKey.cs ================================================ namespace Tubifarry.Download.Clients.Soulseek.Models; public readonly struct DownloadKey(TOuterKey outerKey, TInnerKey innerKey) where TOuterKey : notnull where TInnerKey : notnull { public TOuterKey OuterKey { get; } = outerKey; public TInnerKey InnerKey { get; } = innerKey; public override readonly bool Equals(object? obj) => obj is DownloadKey other && EqualityComparer.Default.Equals(OuterKey, other.OuterKey) && EqualityComparer.Default.Equals(InnerKey, other.InnerKey); public override readonly int GetHashCode() => HashCode.Combine(OuterKey, InnerKey); public static bool operator ==(DownloadKey left, DownloadKey right) => left.Equals(right); public static bool operator !=(DownloadKey left, DownloadKey right) => !(left == right); } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/Models/SlskdDownloadDirectory.cs ================================================ using System.Text.Json; using Tubifarry.Indexers.Soulseek; namespace Tubifarry.Download.Clients.Soulseek.Models; public record SlskdDownloadDirectory(string Directory, int FileCount, List? Files) { public static IEnumerable GetDirectories(JsonElement directoriesElement) { if (directoriesElement.ValueKind != JsonValueKind.Array) yield break; foreach (JsonElement directory in directoriesElement.EnumerateArray()) { yield return new SlskdDownloadDirectory( Directory: directory.TryGetProperty("directory", out JsonElement d) ? d.GetString() ?? string.Empty : string.Empty, FileCount: directory.TryGetProperty("fileCount", out JsonElement fc) ? fc.GetInt32() : 0, Files: directory.TryGetProperty("files", out JsonElement files) ? SlskdDownloadFile.GetFiles(files).ToList() : [] ); } } public List ToSlskdFileDataList() => Files?.Select(f => f.ToSlskdFileData()).ToList() ?? []; public SlskdFolderData CreateFolderData(string username, ISlskdItemsParser slskdItemsParser) => slskdItemsParser.ParseFolderName(Directory) with { Username = username, HasFreeUploadSlot = true, UploadSpeed = 0, LockedFileCount = 0, LockedFiles = [] }; } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/Models/SlskdDownloadFile.cs ================================================ using System.Text.Json; using Tubifarry.Indexers.Soulseek; namespace Tubifarry.Download.Clients.Soulseek.Models; public record SlskdDownloadFile( string Id, string Username, string Direction, string Filename, long Size, long StartOffset, string State, DateTime RequestedAt, DateTime EnqueuedAt, DateTime StartedAt, long BytesTransferred, double AverageSpeed, long BytesRemaining, TimeSpan ElapsedTime, double PercentComplete, TimeSpan RemainingTime, TimeSpan? EndedAt, int? PlaceInQueue = null ) { public static IEnumerable GetFiles(JsonElement filesElement) { if (filesElement.ValueKind != JsonValueKind.Array) yield break; foreach (JsonElement file in filesElement.EnumerateArray()) yield return Parse(file); } public static SlskdDownloadFile? ParseSingle(JsonElement el) => el.ValueKind == JsonValueKind.Object ? Parse(el) : null; private static SlskdDownloadFile Parse(JsonElement file) => new( Id: file.TryGetProperty("id", out JsonElement id) ? id.GetString() ?? string.Empty : string.Empty, Username: file.TryGetProperty("username", out JsonElement username) ? username.GetString() ?? string.Empty : string.Empty, Direction: file.TryGetProperty("direction", out JsonElement direction) ? direction.GetString() ?? string.Empty : string.Empty, Filename: file.TryGetProperty("filename", out JsonElement filename) ? filename.GetString() ?? string.Empty : string.Empty, Size: file.TryGetProperty("size", out JsonElement size) ? size.GetInt64() : 0L, StartOffset: file.TryGetProperty("startOffset", out JsonElement startOffset) ? startOffset.GetInt64() : 0L, State: file.TryGetProperty("state", out JsonElement state) ? state.GetString() ?? string.Empty : string.Empty, RequestedAt: file.TryGetProperty("requestedAt", out JsonElement requestedAt) && DateTime.TryParse(requestedAt.GetString(), out DateTime rat) ? rat : DateTime.MinValue, EnqueuedAt: file.TryGetProperty("enqueuedAt", out JsonElement enqueuedAt) && DateTime.TryParse(enqueuedAt.GetString(), out DateTime eat) ? eat : DateTime.MinValue, StartedAt: file.TryGetProperty("startedAt", out JsonElement startedAt) && DateTime.TryParse(startedAt.GetString(), out DateTime sat) ? sat.ToUniversalTime() : DateTime.MinValue, BytesTransferred: file.TryGetProperty("bytesTransferred", out JsonElement bytesTransferred) ? bytesTransferred.GetInt64() : 0L, AverageSpeed: file.TryGetProperty("averageSpeed", out JsonElement averageSpeed) ? averageSpeed.GetDouble() : 0.0, BytesRemaining: file.TryGetProperty("bytesRemaining", out JsonElement bytesRemaining) ? bytesRemaining.GetInt64() : 0L, ElapsedTime: file.TryGetProperty("elapsedTime", out JsonElement elapsedTime) && TimeSpan.TryParse(elapsedTime.GetString(), out TimeSpan et) ? et : TimeSpan.Zero, PercentComplete: file.TryGetProperty("percentComplete", out JsonElement percentComplete) ? percentComplete.GetDouble() : 0.0, RemainingTime: file.TryGetProperty("remainingTime", out JsonElement remainingTime) && TimeSpan.TryParse(remainingTime.GetString(), out TimeSpan rt) ? rt : TimeSpan.Zero, EndedAt: file.TryGetProperty("endedAt", out JsonElement endedAt) && TimeSpan.TryParse(endedAt.GetString(), out TimeSpan ea) ? ea : null, PlaceInQueue: file.TryGetProperty("placeInQueue", out JsonElement placeInQueue) && placeInQueue.ValueKind != JsonValueKind.Null ? placeInQueue.GetInt32() : null ); public SlskdFileData ToSlskdFileData() { string? ext = Path.GetExtension(Filename); if (!string.IsNullOrEmpty(ext)) ext = ext.TrimStart('.'); return new SlskdFileData( Filename: Filename, BitRate: null, BitDepth: null, Size: Size, Length: null, Extension: ext ?? "", SampleRate: null, Code: 1, IsLocked: false ); } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/Models/SlskdDownloadItem.cs ================================================ using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Download; using NzbDrone.Core.Parser.Model; using System.Text.Json; using Tubifarry.Indexers.Soulseek; namespace Tubifarry.Download.Clients.Soulseek.Models; public class SlskdDownloadItem { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; private readonly Logger _logger; public string ID { get; set; } public List FileData { get; set; } = []; public string? Username { get; set; } public ReleaseInfo ReleaseInfo { get; set; } public event EventHandler? FileStateChanged; private SlskdDownloadDirectory? _slskdDownloadDirectory; private readonly Dictionary _previousFileStates = []; public List PostProcessTasks { get; } = []; public DownloadItemStatus? LastReportedStatus { get; set; } public IReadOnlyDictionary FileStates => _previousFileStates; public SlskdDownloadDirectory? SlskdDownloadDirectory { get => _slskdDownloadDirectory; set { if (_slskdDownloadDirectory == value) return; CompareFileStates(value); _slskdDownloadDirectory = value; } } public SlskdDownloadItem(ReleaseInfo releaseInfo) { _logger = NzbDroneLogger.GetLogger(this); ReleaseInfo = releaseInfo; FileData = JsonSerializer.Deserialize>(ReleaseInfo.Source, _jsonOptions) ?? []; ID = GetStableMD5Id(FileData.Select(file => file.Filename)); _logger.Trace($"Created SlskdDownloadItem with ID: {ID}"); } public static string GetStableMD5Id(IEnumerable filenames) { string combined = string.Join("|", filenames.Order()); byte[] bytes = System.Text.Encoding.UTF8.GetBytes(combined); return BitConverter.ToString(System.Security.Cryptography.MD5.HashData(bytes)).Replace("-", "").ToLowerInvariant(); } private void CompareFileStates(SlskdDownloadDirectory? newDirectory) { if (newDirectory?.Files == null) return; foreach (SlskdDownloadFile file in newDirectory.Files) { if (_previousFileStates.TryGetValue(file.Filename, out SlskdFileState? fileState) && fileState != null) { string previousState = fileState.State; fileState.UpdateFile(file); if (fileState.State != previousState) { _logger.Trace($"State change: {Path.GetFileName(file.Filename)} | {previousState} -> {fileState.State}"); FileStateChanged?.Invoke(this, fileState); } } else { SlskdFileState newFileState = new(file); _previousFileStates.Add(file.Filename, newFileState); DownloadItemStatus initialStatus = SlskdFileState.GetStatus(file.State); if (initialStatus == DownloadItemStatus.Failed) { _logger.Trace($"Initial state is failed: {Path.GetFileName(file.Filename)} | {file.State}"); FileStateChanged?.Invoke(this, newFileState); } } } } public OsPath GetFullFolderPath(OsPath downloadPath) => new(Path.Combine( downloadPath.FullPath, SlskdDownloadDirectory?.Directory .Replace('\\', '/') .TrimEnd('/') .Split('/') .LastOrDefault() ?? "")); } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/Models/SlskdFileState.cs ================================================ using NzbDrone.Core.Download; namespace Tubifarry.Download.Clients.Soulseek.Models; [Flags] public enum TransferStates { None = 0, Queued = 2, Initializing = 4, InProgress = 8, Completed = 16, Succeeded = 32, Cancelled = 64, TimedOut = 128, Errored = 256, Rejected = 512, Aborted = 1024, Locally = 2048, Remotely = 4096, } public class SlskdFileState(SlskdDownloadFile file) { public SlskdDownloadFile File { get; private set; } = file; public int RetryCount { get; private set; } private bool _retried = false; public int MaxRetryCount { get; private set; } = 1; public string State => File.State; public string PreviousState { get; private set; } = "Requested"; public DownloadItemStatus GetStatus() { DownloadItemStatus status = GetStatus(State); if ((status == DownloadItemStatus.Failed && RetryCount < MaxRetryCount) || _retried) return DownloadItemStatus.Warning; return status; } public static DownloadItemStatus GetStatus(string stateStr) { if (Enum.TryParse(stateStr, ignoreCase: true, out TransferStates state)) return GetStatus(state); return DownloadItemStatus.Queued; } public static DownloadItemStatus GetStatus(TransferStates state) => state switch { _ when state.HasFlag(TransferStates.Completed) && state.HasFlag(TransferStates.Succeeded) => DownloadItemStatus.Completed, _ when state.HasFlag(TransferStates.Completed) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.Rejected) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.TimedOut) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.Errored) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.Cancelled) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.Aborted) => DownloadItemStatus.Failed, _ when state.HasFlag(TransferStates.InProgress) => DownloadItemStatus.Downloading, _ when state.HasFlag(TransferStates.Initializing) => DownloadItemStatus.Queued, _ when state.HasFlag(TransferStates.Queued) => DownloadItemStatus.Queued, _ => DownloadItemStatus.Queued, }; public void UpdateFile(SlskdDownloadFile file) { if (!_retried) PreviousState = State; else if (File != null && GetStatus(file.State) == DownloadItemStatus.Failed) PreviousState = "Requested"; File = file; _retried = false; } public void UpdateMaxRetryCount(int maxRetryCount) => MaxRetryCount = maxRetryCount; public void IncrementAttempt() { _retried = true; RetryCount++; } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdApiClient.cs ================================================ using FluentValidation.Results; using NzbDrone.Common.Http; using NzbDrone.Core.Download.Clients; using System.Net; using System.Text.Json; using Tubifarry.Download.Clients.Soulseek.Models; namespace Tubifarry.Download.Clients.Soulseek; public class SlskdApiClient(IHttpClient httpClient) : ISlskdApiClient { private readonly SemaphoreSlim _enqueueLimiter = new(2, 2); public async Task<(List Enqueued, List Failed)> EnqueueDownloadAsync( SlskdProviderSettings settings, string username, IEnumerable<(string Filename, long Size)> files) { await _enqueueLimiter.WaitAsync(); try { string payload = JsonSerializer.Serialize(files.Select(f => new { f.Filename, f.Size })); HttpRequest request = BuildRequest( settings, $"/api/v0/transfers/downloads/{Uri.EscapeDataString(username)}", HttpMethod.Post, payload); HttpResponse response = await httpClient.ExecuteAsync(request); if (response.StatusCode != HttpStatusCode.Created) throw new DownloadClientException($"Enqueue failed for {username}. Status: {response.StatusCode}"); return ([], []); } finally { _enqueueLimiter.Release(); } } public async Task> GetAllTransfersAsync(SlskdProviderSettings settings, bool includeRemoved = false) { string endpoint = "/api/v0/transfers/downloads" + (includeRemoved ? "?includeRemoved=true" : ""); HttpResponse response = await httpClient.ExecuteAsync(BuildRequest(settings, endpoint)); if (response.StatusCode != HttpStatusCode.OK) return []; List result = []; using JsonDocument doc = JsonDocument.Parse(response.Content); foreach (JsonElement userEl in doc.RootElement.EnumerateArray()) { string username = userEl.TryGetProperty("username", out JsonElement u) ? u.GetString() ?? "" : ""; userEl.TryGetProperty("directories", out JsonElement dirsEl); result.Add(new SlskdUserTransfers { Username = username, Directories = SlskdDownloadDirectory.GetDirectories(dirsEl).ToList() }); } return result; } public async Task GetUserTransfersAsync(SlskdProviderSettings settings, string username) { HttpResponse response = await httpClient.ExecuteAsync( BuildRequest(settings, $"/api/v0/transfers/downloads/{Uri.EscapeDataString(username)}")); if (response.StatusCode == HttpStatusCode.NotFound) return null; if (response.StatusCode != HttpStatusCode.OK) return null; using JsonDocument doc = JsonDocument.Parse(response.Content); doc.RootElement.TryGetProperty("directories", out JsonElement dirsEl); return new SlskdUserTransfers { Username = username, Directories = SlskdDownloadDirectory.GetDirectories(dirsEl).ToList() }; } public async Task GetTransferAsync(SlskdProviderSettings settings, string username, string fileId) { HttpResponse response = await httpClient.ExecuteAsync( BuildRequest(settings, $"/api/v0/transfers/downloads/{Uri.EscapeDataString(username)}/{fileId}")); if (response.StatusCode != HttpStatusCode.OK) return null; using JsonDocument doc = JsonDocument.Parse(response.Content); return SlskdDownloadFile.ParseSingle(doc.RootElement); } public async Task GetQueuePositionAsync(SlskdProviderSettings settings, string username, string fileId) { HttpResponse response = await httpClient.ExecuteAsync( BuildRequest(settings, $"/api/v0/transfers/downloads/{Uri.EscapeDataString(username)}/{fileId}/position")); if (response.StatusCode != HttpStatusCode.OK) return null; try { using JsonDocument doc = JsonDocument.Parse(response.Content); if (doc.RootElement.ValueKind == JsonValueKind.Number) return doc.RootElement.GetInt32(); } catch { /* fall through */ } return int.TryParse(response.Content.Trim(), out int position) ? position : null; } public async Task DeleteTransferAsync(SlskdProviderSettings settings, string username, string fileId, bool remove = false) { string endpoint = $"/api/v0/transfers/downloads/{Uri.EscapeDataString(username)}/{fileId}" + (remove ? "?remove=true" : ""); await httpClient.ExecuteAsync(BuildRequest(settings, endpoint, HttpMethod.Delete)); } public async Task DeleteAllCompletedAsync(SlskdProviderSettings settings) => await httpClient.ExecuteAsync( BuildRequest(settings, "/api/v0/transfers/downloads/all/completed", HttpMethod.Delete)); public async Task GetDownloadPathAsync(SlskdProviderSettings settings) { HttpResponse response = await httpClient.ExecuteAsync(BuildRequest(settings, "/api/v0/options")); if (response.StatusCode != HttpStatusCode.OK) return null; using JsonDocument doc = JsonDocument.Parse(response.Content); if (doc.RootElement.TryGetProperty("directories", out JsonElement dirs) && dirs.TryGetProperty("downloads", out JsonElement dl)) return dl.GetString(); return null; } public async Task TestConnectionAsync(SlskdProviderSettings settings) { try { Uri uri = new(settings.BaseUrl); settings.IsLocalhost = uri.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase) || (IPAddress.TryParse(uri.Host, out IPAddress? ip) && IPAddress.IsLoopback(ip)); } catch (UriFormatException ex) { return new ValidationFailure("BaseUrl", $"Invalid BaseUrl format: {ex.Message}"); } try { HttpRequest request = BuildRequest(settings, "/api/v0/application"); request.AllowAutoRedirect = true; request.RequestTimeout = TimeSpan.FromSeconds(30); HttpResponse response = await httpClient.ExecuteAsync(request); if (response.StatusCode != HttpStatusCode.OK) return new ValidationFailure("BaseUrl", $"Unable to connect to Slskd. Status: {response.StatusCode}"); using JsonDocument doc = JsonDocument.Parse(response.Content); JsonElement root = doc.RootElement; if (!root.TryGetProperty("server", out JsonElement serverEl) || !serverEl.TryGetProperty("state", out JsonElement stateEl)) return new ValidationFailure("BaseUrl", "Failed to parse Slskd response: missing 'server' or 'state'."); string? serverState = stateEl.GetString(); if (string.IsNullOrEmpty(serverState) || !serverState.Contains("Connected")) return new ValidationFailure("BaseUrl", $"Slskd server is not connected. State: {serverState}"); settings.DownloadPath = await GetDownloadPathAsync(settings) ?? string.Empty; if (string.IsNullOrEmpty(settings.DownloadPath)) return new ValidationFailure("DownloadPath", "DownloadPath could not be found or is invalid."); return null; } catch (HttpException ex) { return new ValidationFailure("BaseUrl", $"Unable to connect to Slskd: {ex.Message}"); } catch (Exception ex) { return new ValidationFailure(string.Empty, $"Unexpected error: {ex.Message}"); } } public async Task<(List Events, int TotalCount)> GetEventsAsync( SlskdProviderSettings settings, int offset, int limit) { HttpResponse response = await httpClient.ExecuteAsync( BuildRequest(settings, $"/api/v0/events?offset={offset}&limit={limit}")); if (response.StatusCode != HttpStatusCode.OK) return ([], 0); string? totalHeader = response.Headers["X-Total-Count"]; int.TryParse(totalHeader, out int totalCount); using JsonDocument doc = JsonDocument.Parse(response.Content); List records = []; foreach (JsonElement el in doc.RootElement.EnumerateArray()) { records.Add(new SlskdEventRecord { Id = el.TryGetProperty("id", out JsonElement id) && Guid.TryParse(id.GetString(), out Guid g) ? g : Guid.Empty, Timestamp = el.TryGetProperty("timestamp", out JsonElement ts) && DateTime.TryParse(ts.GetString(), out DateTime dt) ? dt : DateTime.MinValue, Type = el.TryGetProperty("type", out JsonElement type) ? type.GetString() ?? "" : "", Data = el.TryGetProperty("data", out JsonElement data) ? data.GetString() ?? "" : "", }); } if (totalCount == 0) totalCount = offset + records.Count; return (records, totalCount); } private HttpRequest BuildRequest(SlskdProviderSettings settings, string endpoint, HttpMethod? method = null, string? content = null) { HttpRequestBuilder builder = new HttpRequestBuilder($"{settings.BaseUrl}{endpoint}") .SetHeader("X-API-KEY", settings.ApiKey) .SetHeader("Accept", "application/json"); if (method != null) builder.Method = method; if (!string.IsNullOrEmpty(content)) { builder.SetHeader("Content-Type", "application/json"); HttpRequest request = builder.Build(); request.SetContent(content); return request; } return builder.Build(); } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; namespace Tubifarry.Download.Clients.Soulseek; public class SlskdClient : DownloadClientBase { private readonly ISlskdDownloadManager _manager; private readonly ISlskdApiClient _apiClient; public override string Name => "Slskd"; public override string Protocol => nameof(SoulseekDownloadProtocol); public SlskdClient( ISlskdDownloadManager manager, ISlskdApiClient apiClient, IConfigService configService, IDiskProvider diskProvider, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _manager = manager; _apiClient = apiClient; } public override async Task Download(RemoteAlbum remoteAlbum, IIndexer indexer) => await _manager.DownloadAsync(remoteAlbum, Definition.Id, Settings); public override IEnumerable GetItems() { DownloadClientItemClientInfo clientInfo = DownloadClientItemClientInfo.FromDownloadClient(this, false); foreach (DownloadClientItem item in _manager.GetItems(Definition.Id, Settings, GetRemoteToLocal())) { item.DownloadClientInfo = clientInfo; yield return item; } } public override void RemoveItem(DownloadClientItem clientItem, bool deleteData) => _manager.RemoveItem(clientItem, deleteData, Definition.Id, Settings); public override DownloadClientInfo GetStatus() => new() { IsLocalhost = Settings.IsLocalhost, OutputRootFolders = [_remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(Settings.DownloadPath))] }; protected override void Test(List failures) => failures.AddIfNotNull(_apiClient.TestConnectionAsync(Settings).GetAwaiter().GetResult()); private OsPath GetRemoteToLocal() => _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(Settings.DownloadPath)); } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdDownloadManager.cs ================================================ using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Download; using NzbDrone.Core.Download.History; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using System.Collections.Concurrent; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Telemetry; using Tubifarry.Download.Clients.Soulseek.Models; using Tubifarry.Indexers.Soulseek; namespace Tubifarry.Download.Clients.Soulseek; internal static class SlskdEventTypes { public const string DownloadDirectoryComplete = "DownloadDirectoryComplete"; public const string DownloadFileComplete = "DownloadFileComplete"; } public class SlskdDownloadManager : ISlskdDownloadManager { private readonly ConcurrentDictionary, SlskdDownloadItem> _downloadMappings = new(); // Adaptive transfer poll times per definition ID private readonly ConcurrentDictionary _lastTransferPollTimes = new(); // Event poll times per definition ID (separate from transfer poll) private readonly ConcurrentDictionary _lastEventPollTimes = new(); // Last-seen event offset per definition ID for incremental polling private readonly ConcurrentDictionary _lastEventOffsets = new(); // Latest settings snapshot per definition ID: used by event-triggered retry callbacks private readonly ConcurrentDictionary _settingsCache = new(); private readonly ISlskdApiClient _apiClient; private readonly IDownloadHistoryService _downloadHistoryService; private readonly ISlskdItemsParser _slskdItemsParser; private readonly IRemotePathMappingService _remotePathMappingService; private readonly IDiskProvider _diskProvider; private readonly ISentryHelper _sentry; private readonly Logger _logger; private readonly SlskdRetryHandler _retryHandler; public SlskdDownloadManager( ISlskdApiClient apiClient, IDownloadHistoryService downloadHistoryService, ISlskdItemsParser slskdItemsParser, IRemotePathMappingService remotePathMappingService, IDiskProvider diskProvider, ISentryHelper sentry, Logger logger) { _apiClient = apiClient; _downloadHistoryService = downloadHistoryService; _slskdItemsParser = slskdItemsParser; _remotePathMappingService = remotePathMappingService; _diskProvider = diskProvider; _sentry = sentry; _logger = logger; _retryHandler = new SlskdRetryHandler(apiClient, sentry, NzbDroneLogger.GetLogger(typeof(SlskdRetryHandler))); } public async Task DownloadAsync(RemoteAlbum remoteAlbum, int definitionId, SlskdProviderSettings settings) { _settingsCache[definitionId] = settings; SlskdDownloadItem item = new(remoteAlbum.Release); _logger.Trace($"Download initiated: {remoteAlbum.Release.Title} | Files: {item.FileData.Count}"); ISpan? span = _sentry.StartSpan("slskd.download", remoteAlbum.Release.Title); _sentry.SetSpanData(span, "album.title", remoteAlbum.Release.Album); _sentry.SetSpanData(span, "album.artist", remoteAlbum.Release.Artist); _sentry.SetSpanData(span, "file_count", item.FileData.Count); try { string username = ExtractUsernameFromPath(remoteAlbum.Release.DownloadUrl); List<(string Filename, long Size)> files = ParseFilesFromSource(remoteAlbum.Release.Source); await _apiClient.EnqueueDownloadAsync(settings, username, files); item.Username = username; SubscribeStateChanges(item, definitionId); AddItem(definitionId, item); _sentry.SetSpanTag(span, "download.id", item.ID); _sentry.FinishSpan(span, SpanStatus.Ok); return item.ID; } catch (Exception ex) { _sentry.FinishSpan(span, ex); throw; } } public IEnumerable GetItems(int definitionId, SlskdProviderSettings settings, OsPath remotePath) { _settingsCache[definitionId] = settings; try { RefreshAsync(definitionId, settings).GetAwaiter().GetResult(); } catch (Exception ex) { _logger.Error(ex, "Failed to update download items from Slskd. Returning cached items."); } TimeSpan? timeout = settings.GetTimeout(); DateTime now = DateTime.UtcNow; foreach (SlskdDownloadItem item in GetItemsForDef(definitionId)) { DownloadClientItem clientItem; try { SlskdStatusResolver.DownloadStatus resolved = SlskdStatusResolver.Resolve(item, timeout, now); clientItem = new() { DownloadId = item.ID, Title = item.ReleaseInfo.Title, CanBeRemoved = true, CanMoveFiles = true, OutputPath = item.GetFullFolderPath(remotePath), Status = resolved.Status, Message = resolved.Message, TotalSize = resolved.TotalSize, RemainingSize = resolved.RemainingSize, RemainingTime = resolved.RemainingTime, }; EmitCompletionSpan(item, resolved); } catch (Exception ex) { _logger.Warn(ex, $"Failed to build DownloadClientItem for {item.ID}. Skipping."); continue; } yield return clientItem; } } public void RemoveItem(DownloadClientItem clientItem, bool deleteData, int definitionId, SlskdProviderSettings settings) { if (!deleteData) return; SlskdDownloadItem? item = GetItem(definitionId, clientItem.DownloadId); if (item == null) return; string? directory = item.SlskdDownloadDirectory?.Directory; _ = RemoveItemFilesAsync(item, settings); RemoveItemFromDict(definitionId, clientItem.DownloadId); if (settings.CleanStaleDirectories && !string.IsNullOrEmpty(directory)) _ = CleanStaleDirectoriesAsync(directory, settings); } private async Task RefreshAsync(int definitionId, SlskdProviderSettings settings) { HashSet activeUsernames = GetActiveUsernames(definitionId); TimeSpan transferInterval = activeUsernames.Count > 0 ? TimeSpan.FromSeconds(3) : TimeSpan.FromSeconds(30); DateTime now = DateTime.UtcNow; DateTime lastTransfer = _lastTransferPollTimes.GetOrAdd(definitionId, DateTime.MinValue); if (now - lastTransfer >= transferInterval) { await PollTransfersAsync(definitionId, settings, activeUsernames); _lastTransferPollTimes[definitionId] = DateTime.UtcNow; } DateTime lastEvent = _lastEventPollTimes.GetOrAdd(definitionId, DateTime.MinValue); if (now - lastEvent >= TimeSpan.FromSeconds(5)) { int offset = _lastEventOffsets.GetOrAdd(definitionId, 0); await PollEventsAsync(definitionId, settings, offset); _lastEventPollTimes[definitionId] = DateTime.UtcNow; } } private async Task PollTransfersAsync(int definitionId, SlskdProviderSettings settings, HashSet activeUsernames) { ConcurrentDictionary currentIdSet = new(); if (!settings.Inclusive && activeUsernames.Count > 0) { await Task.WhenAll(activeUsernames.Select(async username => { SlskdUserTransfers? userTransfers = await _apiClient.GetUserTransfersAsync(settings, username); if (userTransfers != null) ProcessUserTransfers(definitionId, settings, userTransfers, currentIdSet); })); } else { List all = await _apiClient.GetAllTransfersAsync(settings); foreach (SlskdUserTransfers user in all) ProcessUserTransfers(definitionId, settings, user, currentIdSet); } _logger.Debug($"[def={definitionId}] Polled {activeUsernames.Count} users | Tracked: {currentIdSet.Count}"); if (settings.Inclusive) { foreach (SlskdDownloadItem item in GetItemsForDef(definitionId) .Where(i => !currentIdSet.ContainsKey(i.ID) && i.ReleaseInfo.DownloadProtocol == null) .ToList()) { _logger.Trace($"[def={definitionId}] Pruning inclusive item {item.ID} (gone from Slskd)"); RemoveItemFromDict(definitionId, item.ID); } } } private void ProcessUserTransfers( int definitionId, SlskdProviderSettings settings, SlskdUserTransfers userTransfers, ConcurrentDictionary currentIdSet) { foreach (SlskdDownloadDirectory dir in userTransfers.Directories) { string hash = SlskdDownloadItem.GetStableMD5Id(dir.Files?.Select(f => f.Filename) ?? []); currentIdSet.TryAdd(hash, true); SlskdDownloadItem? item = GetItem(definitionId, hash); if (item == null) { _logger.Trace($"[def={definitionId}] Unknown item {hash}: checking history"); DownloadHistory? history = _downloadHistoryService.GetLatestGrab(hash); if (history != null) item = new SlskdDownloadItem(history.Release); else if (settings.Inclusive) item = new SlskdDownloadItem(CreateReleaseInfoFromDirectory(userTransfers.Username, dir)); if (item == null) continue; SubscribeStateChanges(item, definitionId); AddItem(definitionId, item); } item.Username ??= userTransfers.Username; item.SlskdDownloadDirectory = dir; } } private async Task PollEventsAsync(int definitionId, SlskdProviderSettings settings, int offset) { (List events, _) = await _apiClient.GetEventsAsync(settings, offset, 50); if (events.Count == 0) return; foreach (SlskdEventRecord record in events) { try { await HandleEventAsync(definitionId, settings, record); } catch (Exception ex) { _logger.Warn(ex, $"[def={definitionId}] Failed to process event {record.Type} ({record.Id})"); } } _lastEventOffsets[definitionId] = offset + events.Count; } private async Task HandleEventAsync(int definitionId, SlskdProviderSettings settings, SlskdEventRecord record) { if (string.IsNullOrEmpty(record.Data)) return; if (record.Type == SlskdEventTypes.DownloadDirectoryComplete) { using JsonDocument doc = JsonDocument.Parse(record.Data); string remoteDir = doc.RootElement.TryGetProperty("remoteDirectoryName", out JsonElement rdn) ? rdn.GetString() ?? "" : ""; string username = doc.RootElement.TryGetProperty("username", out JsonElement un) ? un.GetString() ?? "" : ""; SlskdDownloadItem? item = GetItemsForDef(definitionId) .FirstOrDefault(i => i.Username == username && string.Equals(i.SlskdDownloadDirectory?.Directory, remoteDir, StringComparison.OrdinalIgnoreCase)); if (item != null) { _logger.Trace($"[def={definitionId}] Event DownloadDirectoryComplete: {remoteDir} by {username}: forcing refresh"); SlskdUserTransfers? userTransfers = await _apiClient.GetUserTransfersAsync(settings, username); if (userTransfers != null) ProcessUserTransfers(definitionId, settings, userTransfers, new ConcurrentDictionary()); } } else if (record.Type == SlskdEventTypes.DownloadFileComplete && _logger.IsTraceEnabled) { using JsonDocument doc = JsonDocument.Parse(record.Data); if (doc.RootElement.TryGetProperty("transfer", out JsonElement transferEl)) { string filename = transferEl.TryGetProperty("filename", out JsonElement fn) ? fn.GetString() ?? "" : ""; string username = transferEl.TryGetProperty("username", out JsonElement un) ? un.GetString() ?? "" : ""; _logger.Trace($"[def={definitionId}] Event DownloadFileComplete: {Path.GetFileName(filename)} by {username}"); } } } private void EmitCompletionSpan(SlskdDownloadItem item, SlskdStatusResolver.DownloadStatus resolved) { bool isTerminal = resolved.Status is DownloadItemStatus.Completed or DownloadItemStatus.Failed; if (!isTerminal || item.LastReportedStatus == resolved.Status) return; item.LastReportedStatus = resolved.Status; int failedCount = item.FileStates.Values.Count(fs => fs.GetStatus() == DownloadItemStatus.Failed); ISpan? span = _sentry.StartSpan("slskd.completion", item.ReleaseInfo.Title); _sentry.SetSpanData(span, "download.id", item.ID); _sentry.SetSpanData(span, "username", item.Username); _sentry.SetSpanData(span, "file_count", item.FileStates.Count); _sentry.SetSpanData(span, "failed_count", failedCount); _sentry.SetSpanData(span, "status", resolved.Status == DownloadItemStatus.Completed ? "completed" : "failed"); if (resolved.Message != null) _sentry.SetSpanData(span, "message", resolved.Message); _sentry.FinishSpan(span, resolved.Status == DownloadItemStatus.Completed ? SpanStatus.Ok : SpanStatus.InternalError); } private void SubscribeStateChanges(SlskdDownloadItem item, int definitionId) { item.FileStateChanged += (sender, fileState) => { if (_settingsCache.TryGetValue(definitionId, out SlskdProviderSettings? s)) _retryHandler.OnFileStateChanged(sender as SlskdDownloadItem, fileState, s); }; } private async Task RemoveItemFilesAsync(SlskdDownloadItem item, SlskdProviderSettings settings) { List files = item.SlskdDownloadDirectory?.Files ?? []; if (files.Count == 0 || item.Username == null) return; await Task.WhenAll(files.Select(async file => { if (SlskdFileState.GetStatus(file.State) != DownloadItemStatus.Completed) { await _apiClient.DeleteTransferAsync(settings, item.Username, file.Id); await Task.Delay(1000); } await _apiClient.DeleteTransferAsync(settings, item.Username, file.Id, remove: true); try { string relativePath = file.Filename; if (relativePath.StartsWith(settings.DownloadPath, StringComparison.OrdinalIgnoreCase)) { relativePath = relativePath.Substring(settings.DownloadPath.Length).TrimStart('/', '\\'); } string localFilePath = _remotePathMappingService .RemapRemoteToLocal(settings.Host, new OsPath(Path.Combine(settings.DownloadPath, relativePath))) .FullPath; if (_diskProvider.FileExists(localFilePath)) { _diskProvider.DeleteFile(localFilePath); _logger.Debug($"Deleted local file: {localFilePath}"); } else { _logger.Trace($"Local file not found or path not accessible, skipping deletion: {Path.GetFileName(file.Filename)}"); } } catch (Exception ex) { _logger.Trace(ex, $"Could not access local file for {Path.GetFileName(file.Filename)}: {ex.Message}"); } _logger.Trace($"Removed transfer {file.Id}"); })); } private async Task CleanStaleDirectoriesAsync(string directoryPath, SlskdProviderSettings settings) { try { string localPath = _remotePathMappingService .RemapRemoteToLocal(settings.Host, new OsPath(Path.Combine(settings.DownloadPath, directoryPath))) .FullPath; await Task.Delay(1000); List all = await _apiClient.GetAllTransfersAsync(settings); bool hasRemaining = all.SelectMany(u => u.Directories) .Any(d => d.Directory.Equals(directoryPath, StringComparison.OrdinalIgnoreCase)); if (hasRemaining) { _logger.Trace($"Directory {directoryPath} still has active downloads: skipping cleanup"); return; } if (_diskProvider.FolderExists(localPath)) { _logger.Debug($"Removing stale directory: {localPath}"); _diskProvider.DeleteFolder(localPath, true); string? parent = Path.GetDirectoryName(localPath); if (!string.IsNullOrEmpty(parent) && _diskProvider.FolderExists(parent) && _diskProvider.FolderEmpty(parent)) { _logger.Info($"Removing empty parent directory: {parent}"); _diskProvider.DeleteFolder(parent, true); } } } catch (Exception ex) { _logger.Error(ex, $"Error cleaning stale directories for path: {directoryPath}"); } } private ReleaseInfo CreateReleaseInfoFromDirectory(string username, SlskdDownloadDirectory dir) { SlskdFolderData folderData = dir.CreateFolderData(username, _slskdItemsParser); SlskdSearchData searchData = new(null, null, false, false, 1, null); IGrouping dirGroup = dir.ToSlskdFileDataList().GroupBy(_ => dir.Directory).First(); AlbumData albumData = _slskdItemsParser.CreateAlbumData(string.Empty, dirGroup, searchData, folderData, null, 0); ReleaseInfo release = albumData.ToReleaseInfo(); release.DownloadProtocol = null; return release; } private static string ExtractUsernameFromPath(string path) { string[] parts = path.TrimEnd('/').Split('/'); return Uri.UnescapeDataString(parts[^1]); } private static List<(string Filename, long Size)> ParseFilesFromSource(string source) { using JsonDocument doc = JsonDocument.Parse(source); return doc.RootElement.EnumerateArray() .Select(el => ( Filename: el.TryGetProperty("Filename", out JsonElement fn) ? fn.GetString() ?? "" : "", Size: el.TryGetProperty("Size", out JsonElement sz) ? sz.GetInt64() : 0L )) .ToList(); } private SlskdDownloadItem? GetItem(int definitionId, string id) => _downloadMappings.TryGetValue(new DownloadKey(definitionId, id), out SlskdDownloadItem? item) ? item : null; private IEnumerable GetItemsForDef(int definitionId) => _downloadMappings .Where(kvp => kvp.Key.OuterKey == definitionId) .Select(kvp => kvp.Value); private void AddItem(int definitionId, SlskdDownloadItem item) => _downloadMappings[new DownloadKey(definitionId, item.ID)] = item; private void RemoveItemFromDict(int definitionId, string id) => _downloadMappings.TryRemove(new DownloadKey(definitionId, id), out _); private HashSet GetActiveUsernames(int definitionId) => [.. GetItemsForDef(definitionId) .Where(i => i.Username != null) .Select(i => i.Username!)]; } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using System.Text.RegularExpressions; namespace Tubifarry.Download.Clients.Soulseek { internal class SlskdProviderSettingsValidator : AbstractValidator { public SlskdProviderSettingsValidator() { // Base URL validation RuleFor(c => c.BaseUrl) .ValidRootUrl() .Must(url => !url.EndsWith('/')) .WithMessage("Base URL must not end with a slash ('/')."); // API Key validation RuleFor(c => c.ApiKey) .NotEmpty() .WithMessage("API Key is required."); // Timeout validation (only if it has a value) RuleFor(c => c.Timeout) .GreaterThanOrEqualTo(0.1) .WithMessage("Timeout must be at least 0.1 hours.") .When(c => c.Timeout.HasValue); // RetryAttempts validation RuleFor(c => c.RetryAttempts) .InclusiveBetween(0, 10) .WithMessage("Retry attempts must be between 0 and 10."); } } public partial class SlskdProviderSettings : IProviderConfig { private static readonly SlskdProviderSettingsValidator Validator = new(); private string? _host; [FieldDefinition(0, Label = "URL", Type = FieldType.Url, Placeholder = "http://localhost:5030", HelpText = "The URL of your Slskd instance.")] public string BaseUrl { get; set; } = "http://localhost:5030"; [FieldDefinition(1, Label = "API Key", Type = FieldType.Textbox, Privacy = PrivacyLevel.ApiKey, HelpText = "The API key for your Slskd instance. You can find or set this in the Slskd's settings under 'Options'.", Placeholder = "Enter your API key")] public string ApiKey { get; set; } = string.Empty; [FieldDefinition(3, Label = "Timeout", Type = FieldType.Textbox, HelpText = "Specify the maximum time to wait for a response from the Slskd instance before timing out. Fractional values are allowed (e.g., 1.5 for 1 hour and 30 minutes). Set leave blank for no timeout.", Unit = "hours", Advanced = true, Placeholder = "Enter timeout in hours")] public double? Timeout { get; set; } [FieldDefinition(4, Label = "Retry Attempts", Type = FieldType.Number, HelpText = "The number of times to retry downloading a file if it fails.", Advanced = true, Placeholder = "Enter retry attempts")] public int RetryAttempts { get; set; } = 1; [FieldDefinition(5, Label = "Inclusive", Type = FieldType.Checkbox, HelpText = "Include all downloads made in Slskd, or only the ones initialized by this Lidarr instance.", Advanced = true)] public bool Inclusive { get; set; } [FieldDefinition(6, Label = "Clean Directories", Type = FieldType.Checkbox, HelpText = "After importing, remove stale directories.", Advanced = true)] public bool CleanStaleDirectories { get; set; } [FieldDefinition(98, Label = "Is Fetched remote", Type = FieldType.Checkbox, Hidden = HiddenType.Hidden)] public bool IsRemotePath { get; set; } [FieldDefinition(99, Label = "Host", Type = FieldType.Textbox, Hidden = HiddenType.Hidden)] public string Host { get => _host ??= (HostRegex().Match(BaseUrl) is { Success: true } match) ? match.Groups[1].Value : BaseUrl; set { } } public bool IsLocalhost { get; set; } public string DownloadPath { get; set; } = string.Empty; public TimeSpan? GetTimeout() => Timeout == null ? null : TimeSpan.FromHours(Timeout.Value); public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); [GeneratedRegex(@"^(?:https?:\/\/)?([^\/:\?]+)(?::\d+)?(?:\/|$)", RegexOptions.Compiled)] private static partial Regex HostRegex(); } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdRetryHandler.cs ================================================ using NLog; using NzbDrone.Core.Download; using System.Text.Json; using Tubifarry.Core.Telemetry; using Tubifarry.Download.Clients.Soulseek.Models; namespace Tubifarry.Download.Clients.Soulseek; public class SlskdRetryHandler(ISlskdApiClient apiClient, ISentryHelper sentry, Logger logger) { private readonly ISlskdApiClient _apiClient = apiClient; private readonly ISentryHelper _sentry = sentry; private readonly Logger _logger = logger; public void OnFileStateChanged(SlskdDownloadItem? item, SlskdFileState fileState, SlskdProviderSettings settings) { fileState.UpdateMaxRetryCount(settings.RetryAttempts); if (fileState.GetStatus() != DownloadItemStatus.Warning) return; if (item == null) return; _logger.Trace($"Retry triggered: {Path.GetFileName(fileState.File.Filename)} | State: {fileState.State} | Attempt: {fileState.RetryCount + 1}/{fileState.MaxRetryCount}"); _ = RetryDownloadAsync(item, fileState, settings); } private async Task RetryDownloadAsync(SlskdDownloadItem item, SlskdFileState fileState, SlskdProviderSettings settings) { ISpan? span = _sentry.StartSpan("slskd.retry", Path.GetFileName(fileState.File.Filename)); _sentry.SetSpanData(span, "file.name", Path.GetFileName(fileState.File.Filename)); _sentry.SetSpanData(span, "retry.attempt", fileState.RetryCount + 1); try { using JsonDocument doc = JsonDocument.Parse(item.ReleaseInfo.Source); JsonElement matchingEl = doc.RootElement.EnumerateArray() .FirstOrDefault(x => x.TryGetProperty("Filename", out JsonElement fn) && fn.GetString() == fileState.File.Filename); if (matchingEl.ValueKind == JsonValueKind.Undefined) { _sentry.FinishSpan(span, SpanStatus.NotFound); return; } long size = matchingEl.TryGetProperty("Size", out JsonElement sz) ? sz.GetInt64() : 0L; string username = item.Username ?? ExtractUsernameFromPath(item.ReleaseInfo.DownloadUrl); await _apiClient.EnqueueDownloadAsync(settings, username, [(fileState.File.Filename, size)]); _logger.Trace($"Retry enqueued: {Path.GetFileName(fileState.File.Filename)}"); _sentry.FinishSpan(span, SpanStatus.Ok); } catch (Exception ex) { _logger.Error(ex, $"Failed to retry download for file: {fileState.File.Filename}"); _sentry.FinishSpan(span, ex); } finally { fileState.IncrementAttempt(); } } private static string ExtractUsernameFromPath(string path) { string[] parts = path.TrimEnd('/').Split('/'); return Uri.UnescapeDataString(parts[^1]); } } ================================================ FILE: Tubifarry/Download/Clients/Soulseek/SlskdStatusResolver.cs ================================================ using NzbDrone.Core.Download; using Tubifarry.Download.Clients.Soulseek.Models; namespace Tubifarry.Download.Clients.Soulseek; public static class SlskdStatusResolver { public record DownloadStatus( DownloadItemStatus Status, string? Message, long TotalSize, long RemainingSize, TimeSpan? RemainingTime ); public static DownloadStatus Resolve(SlskdDownloadItem item, TimeSpan? timeout, DateTime utcNow) { if (item.SlskdDownloadDirectory?.Files == null) return new(DownloadItemStatus.Queued, null, 0, 0, null); IReadOnlyList files = item.SlskdDownloadDirectory.Files; long totalSize = 0, remainingSize = 0, totalSpeed = 0; bool anyActive = false, anyIncomplete = false, allIncompleteRemoteQueued = true; DateTime lastActivity = DateTime.MinValue; foreach (SlskdDownloadFile f in files) { totalSize += f.Size; remainingSize += f.BytesRemaining; DownloadItemStatus fs = SlskdFileState.GetStatus(f.State); if (fs == DownloadItemStatus.Completed) continue; anyIncomplete = true; if (fs == DownloadItemStatus.Downloading) { anyActive = true; totalSpeed += (long)f.AverageSpeed; } else if (fs == DownloadItemStatus.Queued) { anyActive = true; } // Inactivity timestamp: max of enqueued / started / (started + elapsed) DateTime t3 = f.StartedAt + f.ElapsedTime; DateTime latest = f.EnqueuedAt > f.StartedAt ? f.EnqueuedAt : f.StartedAt; if (t3 > latest) latest = t3; if (latest > lastActivity) lastActivity = latest; // All-stuck check: short-circuit once one file is NOT stuck if (allIncompleteRemoteQueued) { bool stuckRemote = timeout.HasValue && Enum.TryParse(f.State, ignoreCase: true, out TransferStates ts) && ts.HasFlag(TransferStates.Queued) && ts.HasFlag(TransferStates.Remotely) && (utcNow - f.EnqueuedAt) > timeout.Value; if (!stuckRemote) allIncompleteRemoteQueued = false; } } bool allStuckInRemoteQueue = anyIncomplete && allIncompleteRemoteQueued; int totalFileCount = 0, failedCount = 0, completedCount = 0; bool anyWarning = false, anyPaused = false, anyDownloadingState = false; List failedFileNames = []; foreach (SlskdFileState fs in item.FileStates.Values) { totalFileCount++; DownloadItemStatus s = fs.GetStatus(); switch (s) { case DownloadItemStatus.Completed: completedCount++; break; case DownloadItemStatus.Failed: failedCount++; failedFileNames.Add(Path.GetFileName(fs.File.Filename)); break; case DownloadItemStatus.Warning: anyWarning = true; break; case DownloadItemStatus.Paused: anyPaused = true; break; case DownloadItemStatus.Downloading: anyDownloadingState = true; break; } } DownloadItemStatus status; string? message = null; if (allStuckInRemoteQueue && !anyActive) { status = DownloadItemStatus.Failed; message = "All files stuck in remote queue past timeout."; } else if (!anyActive && anyIncomplete) { status = timeout.HasValue && (utcNow - lastActivity) > timeout.Value * 2 ? DownloadItemStatus.Failed : DownloadItemStatus.Queued; } else if (totalFileCount > 0 && (double)failedCount / totalFileCount * 100 > 20) { status = DownloadItemStatus.Failed; message = $"Downloading {failedCount} files failed: {string.Join(", ", failedFileNames)}"; } else if (failedCount != 0) { status = DownloadItemStatus.Warning; message = $"Downloading {failedCount} files failed: {string.Join(", ", failedFileNames)}"; } else if (totalFileCount > 0 && completedCount == totalFileCount) { status = item.PostProcessTasks.Any(t => !t.IsCompleted) ? DownloadItemStatus.Downloading : DownloadItemStatus.Completed; } else if (anyPaused) { status = DownloadItemStatus.Paused; } else if (anyWarning) { status = DownloadItemStatus.Warning; message = "Some files failed. Retrying download..."; } else if (anyDownloadingState) { status = DownloadItemStatus.Downloading; } else { status = DownloadItemStatus.Queued; } TimeSpan? remainingTime = totalSpeed > 0 ? TimeSpan.FromSeconds(remainingSize / (double)totalSpeed) : null; return new(status, message, totalSize, remainingSize, remainingTime); } } ================================================ FILE: Tubifarry/Download/Clients/SubSonic/SubSonicClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using System.Text.Json; using Tubifarry.Core.Utilities; using Tubifarry.Download.Base; using Tubifarry.Indexers.SubSonic; namespace Tubifarry.Download.Clients.SubSonic { /// /// SubSonic download client for downloading music from SubSonic servers /// Integrates with SubSonic API to download tracks and albums /// public class SubSonicClient : DownloadClientBase { private readonly ISubSonicDownloadManager _downloadManager; private readonly INamingConfigService _namingService; private readonly IEnumerable _requestInterceptors; public SubSonicClient( ISubSonicDownloadManager downloadManager, IConfigService configService, IDiskProvider diskProvider, INamingConfigService namingConfigService, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, IEnumerable requestInterceptors, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _downloadManager = downloadManager; _requestInterceptors = requestInterceptors; _namingService = namingConfigService; } public override string Name => "SubSonic"; public override string Protocol => nameof(SubSonicDownloadProtocol); public new SubSonicProviderSettings Settings => base.Settings; public override Task Download(RemoteAlbum remoteAlbum, IIndexer indexer) => _downloadManager.Download(remoteAlbum, indexer, _namingService.GetConfig(), this); public override IEnumerable GetItems() => _downloadManager.GetItems(); public override void RemoveItem(DownloadClientItem item, bool deleteData) { if (deleteData) DeleteItemData(item); _downloadManager.RemoveItem(item); } public override DownloadClientInfo GetStatus() => new() { IsLocalhost = false, OutputRootFolders = [new OsPath(Settings.DownloadPath)] }; protected override void Test(List failures) { // Test download path if (!_diskProvider.FolderExists(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path does not exist")); return; } if (!_diskProvider.FolderWritable(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path is not writable")); } // Test SubSonic connection try { string baseUrl = Settings.ServerUrl.TrimEnd('/'); System.Text.StringBuilder urlBuilder = new($"{baseUrl}/rest/ping.view"); SubSonicAuthHelper.AppendAuthParameters(urlBuilder, Settings.Username, Settings.Password, Settings.UseTokenAuth); urlBuilder.Append("&f=json"); string testUrl = urlBuilder.ToString(); BaseHttpClient httpClient = new(Settings.ServerUrl, _requestInterceptors, TimeSpan.FromSeconds(Settings.RequestTimeout)); using HttpRequestMessage request = httpClient.CreateRequest(HttpMethod.Get, testUrl); _logger.Trace("Testing SubSonic connection to: {BaseUrl}", Settings.ServerUrl); HttpResponseMessage response = httpClient.SendAsync(request, CancellationToken.None).GetAwaiter().GetResult(); response.EnsureSuccessStatusCode(); string responseContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); SubSonicPingResponse? responseWrapper = JsonSerializer.Deserialize( responseContent, IndexerParserHelper.StandardJsonOptions); if (responseWrapper?.SubsonicResponse != null) { SubSonicPingData pingResponse = responseWrapper.SubsonicResponse; if (pingResponse.Status == "ok") { _logger.Debug($"Successfully connected to SubSonic server as {Settings.Username} (API version: {pingResponse.Version})"); return; } else if (pingResponse.Error != null) { int errorCode = pingResponse.Error.Code; string errorMsg = pingResponse.Error.Message; if (errorCode == 40 || errorCode == 41) // Authentication errors { failures.Add(new ValidationFailure("Username", $"Authentication failed: {errorMsg}. Check your username and password.")); } else { failures.Add(new ValidationFailure("ServerUrl", $"SubSonic API error: {errorMsg}")); } return; } } failures.Add(new ValidationFailure("ServerUrl", "Failed to connect to SubSonic server. Check the server URL and credentials.")); } catch (HttpRequestException ex) { _logger.Error(ex, "HTTP error testing SubSonic connection"); failures.Add(new ValidationFailure("ServerUrl", $"Cannot connect to SubSonic server: {ex.Message}. Check the server URL.")); } catch (Exception ex) { _logger.Error(ex, "Error testing SubSonic connection"); failures.Add(new ValidationFailure("ServerUrl", $"Error connecting to SubSonic: {ex.Message}")); } } } } ================================================ FILE: Tubifarry/Download/Clients/SubSonic/SubSonicDownloadManager.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Tubifarry.Download.Base; namespace Tubifarry.Download.Clients.SubSonic { public interface ISubSonicDownloadManager : IBaseDownloadManager { } /// /// Manager for SubSonic downloads, handles creating and managing download requests /// public class SubSonicDownloadManager(IEnumerable requestInterceptors, Logger logger) : BaseDownloadManager(logger), ISubSonicDownloadManager { private readonly IEnumerable _requestInterceptors = requestInterceptors; protected override async Task CreateDownloadRequest( RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, SubSonicClient provider) { string baseUrl = provider.Settings.ServerUrl.TrimEnd('/'); string downloadUrl = remoteAlbum.Release.DownloadUrl; if (!downloadUrl.StartsWith(baseUrl)) _logger.Warn("The expected URL does not match the configured API URL."); bool isTrack = downloadUrl.Contains("/track/"); string itemId = ExtractIdFromUrl(downloadUrl); _logger.Trace($"Type from URL: {(isTrack ? "Track" : "Album")}, Extracted ID: {itemId}"); SubSonicDownloadOptions options = new() { Handler = _requesthandler, DownloadPath = provider.Settings.DownloadPath, BaseUrl = baseUrl, Username = provider.Settings.Username, Password = provider.Settings.Password, UseTokenAuth = provider.Settings.UseTokenAuth, MaxDownloadSpeed = provider.Settings.MaxDownloadSpeed * 1024, // Convert KB/s to bytes/s ConnectionRetries = provider.Settings.ConnectionRetries, RequestTimeout = provider.Settings.RequestTimeout, NamingConfig = namingConfig, RequestInterceptors = _requestInterceptors, DelayBetweenAttemps = TimeSpan.FromSeconds(2), NumberOfAttempts = (byte)provider.Settings.ConnectionRetries, ClientInfo = DownloadClientItemClientInfo.FromDownloadClient(provider, false), IsTrack = isTrack, ItemId = itemId, PreferredFormat = (PreferredFormatEnum)provider.Settings.PreferredFormat, MaxBitRate = provider.Settings.MaxBitRate }; _requesthandler.MaxParallelism = provider.Settings.MaxParallelDownloads; return await Task.FromResult(new SubSonicDownloadRequest(remoteAlbum, options)); } private static string ExtractIdFromUrl(string url) { if (string.IsNullOrEmpty(url)) return string.Empty; // The pattern is: {baseUrl}://album/{id} or {baseUrl}://track/{id} int albumPos = url.IndexOf("/album/", StringComparison.OrdinalIgnoreCase); if (albumPos >= 0) return url[(albumPos + "/album/".Length)..]; int trackPos = url.IndexOf("/track/", StringComparison.OrdinalIgnoreCase); if (trackPos >= 0) return url[(trackPos + "/track/".Length)..]; // Fallback: try to extract from path string[] parts = url.Split('/', StringSplitOptions.RemoveEmptyEntries); if (parts.Length > 0) return parts[^1]; return string.Empty; } } } ================================================ FILE: Tubifarry/Download/Clients/SubSonic/SubSonicDownloadOptions.cs ================================================ using Tubifarry.Download.Base; namespace Tubifarry.Download.Clients.SubSonic { /// /// Download options specific to SubSonic downloads /// public record SubSonicDownloadOptions : BaseDownloadOptions { /// /// SubSonic username for authentication /// public string Username { get; set; } = string.Empty; /// /// SubSonic password for authentication /// public string Password { get; set; } = string.Empty; /// /// Whether to use token-based authentication /// public bool UseTokenAuth { get; set; } = true; /// /// Preferred audio format for transcoding /// public PreferredFormatEnum PreferredFormat { get; set; } = PreferredFormatEnum.Raw; /// /// Maximum bit rate in kbps (0 for original quality) /// public int MaxBitRate { get; set; } = 0; public SubSonicDownloadOptions() : base() { } protected SubSonicDownloadOptions(SubSonicDownloadOptions options) : base(options) { Username = options.Username; Password = options.Password; UseTokenAuth = options.UseTokenAuth; PreferredFormat = options.PreferredFormat; MaxBitRate = options.MaxBitRate; } } } ================================================ FILE: Tubifarry/Download/Clients/SubSonic/SubSonicDownloadRequest.cs ================================================ using DownloadAssistant.Options; using DownloadAssistant.Requests; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using System.Text; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; using Tubifarry.Download.Base; using Tubifarry.Indexers.SubSonic; namespace Tubifarry.Download.Clients.SubSonic { /// /// SubSonic download request handling track and album downloads /// public class SubSonicDownloadRequest : BaseDownloadRequest { private readonly BaseHttpClient _httpClient; private SubSonicAlbumFull? _currentAlbum; public SubSonicDownloadRequest(RemoteAlbum remoteAlbum, SubSonicDownloadOptions? options) : base(remoteAlbum, options) { _httpClient = new BaseHttpClient( Options.BaseUrl, Options.RequestInterceptors, TimeSpan.FromSeconds(Options.RequestTimeout)); _requestContainer.Add(new OwnRequest(async (token) => { try { await ProcessDownloadAsync(token); return true; } catch (Exception ex) { LogAndAppendMessage($"Error processing download: {ex.Message}", LogLevel.Error); throw; } }, new RequestOptions() { CancellationToken = Token, DelayBetweenAttemps = Options.DelayBetweenAttemps, NumberOfAttempts = Options.NumberOfAttempts, Priority = RequestPriority.Low, Handler = Options.Handler })); } protected override async Task ProcessDownloadAsync(CancellationToken token) { _logger.Trace($"Processing {(Options.IsTrack ? "track" : "album")}: {ReleaseInfo.Title}"); if (Options.IsTrack) await ProcessSingleTrackAsync(Options.ItemId, token); else await ProcessAlbumAsync(Options.ItemId, token); } private async Task ProcessSingleTrackAsync(string songId, CancellationToken token) { _logger.Trace($"Processing single track with ID: {songId}"); SubSonicSearchSong track = await GetSongAsync(songId, token); if (!string.IsNullOrEmpty(track.AlbumId)) _currentAlbum = await TryGetAlbumAsync(track.AlbumId, token); await TryDownloadAlbumCoverAsync(track.CoverArt, token); QueueTrackDownload(track, token); _requestContainer.Add(_trackContainer); } private async Task ProcessAlbumAsync(string albumId, CancellationToken token) { _logger.Trace($"Processing album with ID: {albumId}"); _currentAlbum = await GetAlbumAsync(albumId, token); if ((_currentAlbum.Songs?.Count ?? 0) == 0) throw new Exception("No tracks found in album"); _expectedTrackCount = _currentAlbum.Songs!.Count; _logger.Trace($"Found {_expectedTrackCount} tracks in album: {_currentAlbum.Name}"); await TryDownloadAlbumCoverAsync(_currentAlbum.CoverArt, token); for (int i = 0; i < _currentAlbum.Songs.Count; i++) { SubSonicSearchSong track = _currentAlbum.Songs[i]; try { QueueTrackDownload(track, token); _logger.Trace($"Track {i + 1}/{_expectedTrackCount} queued: {track.Title}"); } catch (Exception ex) { LogAndAppendMessage($"Track {i + 1}/{_expectedTrackCount} failed: {track.Title} - {ex.Message}", LogLevel.Error); } } _requestContainer.Add(_trackContainer); } private async Task GetSongAsync(string songId, CancellationToken token) { try { string response = await ExecuteApiRequestAsync("getSong.view", songId, token); SubSonicSongResponseWrapper? responseWrapper = JsonSerializer.Deserialize(response, IndexerParserHelper.StandardJsonOptions); ValidateApiResponse(responseWrapper?.SubsonicResponse); if (responseWrapper!.SubsonicResponse!.Song == null) throw new Exception("Song data not found in response"); return responseWrapper.SubsonicResponse.Song; } catch (Exception ex) { throw new Exception($"Failed to get song details: {ex.Message}", ex); } } private async Task GetAlbumAsync(string albumId, CancellationToken token) { try { string response = await ExecuteApiRequestAsync("getAlbum.view", albumId, token); SubSonicAlbumResponseWrapper? responseWrapper = JsonSerializer.Deserialize(response, IndexerParserHelper.StandardJsonOptions); ValidateApiResponse(responseWrapper?.SubsonicResponse); if (responseWrapper!.SubsonicResponse!.Album == null) throw new Exception("Album data not found in response"); return responseWrapper.SubsonicResponse.Album; } catch (Exception ex) { throw new Exception($"Failed to get album details: {ex.Message}", ex); } } private async Task TryGetAlbumAsync(string albumId, CancellationToken token) { try { return await GetAlbumAsync(albumId, token); } catch (Exception ex) { _logger.Warn(ex, $"Failed to get album details for albumId: {albumId}"); return null; } } private async Task TryDownloadAlbumCoverAsync(string? coverArtId, CancellationToken token) { if (string.IsNullOrEmpty(coverArtId)) return; try { string coverUrl = BuildCoverArtUrl(coverArtId); using HttpResponseMessage response = await _httpClient.GetAsync(coverUrl, token); response.EnsureSuccessStatusCode(); _albumCover = await response.Content.ReadAsByteArrayAsync(token); _logger.Trace($"Downloaded album cover: {_albumCover.Length} bytes"); } catch (Exception ex) { _logger.Warn(ex, "Failed to download album cover"); _albumCover = null; } } private async Task ExecuteApiRequestAsync(string endpoint, string id, CancellationToken token) { string url = BuildApiUrl(endpoint, id); using HttpRequestMessage request = _httpClient.CreateRequest(HttpMethod.Get, url); using HttpResponseMessage response = await _httpClient.SendAsync(request, token); response.EnsureSuccessStatusCode(); return await response.Content.ReadAsStringAsync(token); } private static void ValidateApiResponse(SubSonicItemResponse? response) { if (response?.Status != "ok") { if (response?.Error != null) throw new Exception($"SubSonic API error: {response.Error.Message ?? "Unknown error"}"); throw new Exception("SubSonic API returned an error status"); } } private string BuildApiUrl(string endpoint, string id) { StringBuilder urlBuilder = new(); urlBuilder.Append($"{Options.BaseUrl.TrimEnd('/')}/rest/{endpoint}"); urlBuilder.Append($"?id={Uri.EscapeDataString(id)}"); AppendStandardApiParameters(urlBuilder); return urlBuilder.ToString(); } private string BuildStreamUrl(string songId) { StringBuilder urlBuilder = new(); urlBuilder.Append($"{Options.BaseUrl.TrimEnd('/')}/rest/stream.view"); urlBuilder.Append($"?id={Uri.EscapeDataString(songId)}"); AppendStandardApiParameters(urlBuilder); if (Options.MaxBitRate > 0) urlBuilder.Append($"&maxBitRate={Options.MaxBitRate}"); if (Options.PreferredFormat != PreferredFormatEnum.Raw) urlBuilder.Append($"&format={Options.PreferredFormat.ToString().ToLower()}"); return urlBuilder.ToString(); } private string BuildCoverArtUrl(string coverArtId) { StringBuilder urlBuilder = new(); urlBuilder.Append($"{Options.BaseUrl.TrimEnd('/')}/rest/getCoverArt.view"); urlBuilder.Append($"?id={Uri.EscapeDataString(coverArtId)}"); AppendStandardApiParameters(urlBuilder); return urlBuilder.ToString(); } private void AppendStandardApiParameters(StringBuilder urlBuilder) { SubSonicAuthHelper.AppendAuthParameters(urlBuilder, Options.Username, Options.Password, Options.UseTokenAuth); urlBuilder.Append("&f=json"); } private void QueueTrackDownload(SubSonicSearchSong track, CancellationToken token) { string streamUrl = BuildStreamUrl(track.Id); Track trackMetadata = CreateTrackFromSubSonicData(track); Album albumMetadata = CreateAlbumFromSubSonicData(track, _currentAlbum); string fileName = BuildTrackFilename(trackMetadata, albumMetadata); LoadRequest downloadRequest = CreateDownloadRequest(streamUrl, fileName, token); OwnRequest postProcessRequest = CreatePostProcessRequest(track, downloadRequest, fileName, token); downloadRequest.TrySetSubsequentRequest(postProcessRequest); postProcessRequest.TrySetIdle(); _trackContainer.Add(downloadRequest); _requestContainer.Add(postProcessRequest); } private LoadRequest CreateDownloadRequest(string streamUrl, string fileName, CancellationToken token) => new(streamUrl, new LoadRequestOptions() { CancellationToken = token, CreateSpeedReporter = true, SpeedReporterTimeout = 1000, Priority = RequestPriority.Normal, MaxBytesPerSecond = Options.MaxDownloadSpeed, DelayBetweenAttemps = Options.DelayBetweenAttemps, Filename = fileName, AutoStart = true, DestinationPath = _destinationPath.FullPath, Handler = Options.Handler, DeleteFilesOnFailure = true, RequestFailed = (_, __) => LogAndAppendMessage($"Download failed: {fileName}", LogLevel.Error), WriteMode = WriteMode.AppendOrTruncate, }); private OwnRequest CreatePostProcessRequest(SubSonicSearchSong track, LoadRequest downloadRequest, string fileName, CancellationToken token) => new( (t) => PostProcessTrackAsync(track, downloadRequest, t), new RequestOptions() { AutoStart = false, Priority = RequestPriority.High, DelayBetweenAttemps = Options.DelayBetweenAttemps, Handler = Options.Handler, CancellationToken = token, RequestFailed = (_, __) => LogAndAppendMessage($"Post-processing failed: {fileName}", LogLevel.Error) }); private async Task PostProcessTrackAsync(SubSonicSearchSong trackInfo, LoadRequest request, CancellationToken token) { string trackPath = request.Destination; await Task.Delay(100, token); if (!File.Exists(trackPath)) { _logger.Error($"Track file not found after download: {trackPath}"); return false; } try { AudioMetadataHandler audioData = new(trackPath) { AlbumCover = _albumCover }; AudioFormat detectedFormat = AudioFormatHelper.GetAudioCodecFromExtension(trackPath); if (!AudioMetadataHandler.SupportsMetadataEmbedding(detectedFormat)) { _logger.Warn($"Skipping metadata embedding for {detectedFormat} format. Not supported: {Path.GetFileName(trackPath)}"); return true; } Album album = CreateAlbumFromSubSonicData(trackInfo, _currentAlbum); Track track = CreateTrackFromSubSonicData(trackInfo); if (!audioData.TryEmbedMetadata(album, track)) { _logger.Warn($"Failed to embed metadata for: {Path.GetFileName(trackPath)}"); return false; } _logger.Trace($"Successfully processed track: {Path.GetFileName(trackPath)}"); return true; } catch (Exception ex) { LogAndAppendMessage($"Post-processing failed for {Path.GetFileName(trackPath)}: {ex.Message}", LogLevel.Error); return false; } } private Album CreateAlbumFromSubSonicData(SubSonicSearchSong track, SubSonicAlbumFull? albumInfo) { string albumTitle = albumInfo?.Name ?? track.DisplayAlbum ?? ReleaseInfo.Album ?? _remoteAlbum.Albums?.FirstOrDefault()?.Title ?? "Unknown Album"; string artistName = albumInfo?.Artist ?? track.Artist ?? ReleaseInfo.Artist ?? _remoteAlbum.Artist?.Name ?? "Unknown Artist"; DateTime releaseDate = DetermineReleaseDate(albumInfo, track); return new Album { Title = albumTitle, ReleaseDate = releaseDate, Artist = new LazyLoaded(new Artist { Name = artistName }), AlbumReleases = new LazyLoaded>(new List { new() { TrackCount = albumInfo?.SongCount ?? 0, Title = albumTitle, } }), Genres = _remoteAlbum.Albums?.FirstOrDefault()?.Genres, }; } private DateTime DetermineReleaseDate(SubSonicAlbumFull? albumInfo, SubSonicSearchSong track) { int? year = albumInfo?.Year ?? track.Year; if (year > 1900 && year.Value <= DateTime.Now.Year) return new DateTime(year.Value, 1, 1); if (albumInfo?.Created != null) return albumInfo.Created.Value; return ReleaseInfo.PublishDate; } private static Track CreateTrackFromSubSonicData(SubSonicSearchSong track) => new() { Title = track.Title, TrackNumber = track.Track?.ToString() ?? "1", AbsoluteTrackNumber = track.Track ?? 1, Duration = track.Duration * 1000, // Convert seconds to milliseconds Artist = new LazyLoaded(new Artist { Name = track.Artist }) }; } } ================================================ FILE: Tubifarry/Download/Clients/SubSonic/SubSonicProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; namespace Tubifarry.Download.Clients.SubSonic { public class SubSonicProviderSettingsValidator : AbstractValidator { public SubSonicProviderSettingsValidator() { // Validate DownloadPath RuleFor(x => x.DownloadPath) .IsValidPath() .WithMessage("Download path must be a valid directory."); // Validate ServerUrl RuleFor(x => x.ServerUrl) .NotEmpty().WithMessage("Server URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Server URL must be a valid URL."); // Validate Username RuleFor(x => x.Username) .NotEmpty().WithMessage("Username is required."); // Validate Password RuleFor(x => x.Password) .NotEmpty().WithMessage("Password is required."); // Validate ConnectionRetries RuleFor(x => x.ConnectionRetries) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(10) .WithMessage("Connection retries must be between 1 and 10."); // Validate MaxParallelDownloads RuleFor(x => x.MaxParallelDownloads) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(5) .WithMessage("Max parallel downloads must be between 1 and 5."); // Validate MaxDownloadSpeed RuleFor(x => x.MaxDownloadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Max download speed must be greater than or equal to 0.") .LessThanOrEqualTo(100_000) .WithMessage("Max download speed must be less than or equal to 100 MB/s."); // Validate RequestTimeout RuleFor(x => x.RequestTimeout) .GreaterThanOrEqualTo(10) .LessThanOrEqualTo(300) .WithMessage("Request timeout must be between 10 and 300 seconds."); } } /// /// Configuration settings for the SubSonic download client /// public class SubSonicProviderSettings : IProviderConfig { private static readonly SubSonicProviderSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Directory where downloaded files will be saved")] public string DownloadPath { get; set; } = string.Empty; [FieldDefinition(1, Label = "Server URL", Type = FieldType.Textbox, HelpText = "URL of your SubSonic server", Placeholder = "https://music.example.com")] public string ServerUrl { get; set; } = string.Empty; [FieldDefinition(2, Label = "Username", Type = FieldType.Textbox, HelpText = "Your SubSonic username")] public string Username { get; set; } = string.Empty; [FieldDefinition(3, Label = "Password", Type = FieldType.Password, HelpText = "Your SubSonic password", Privacy = PrivacyLevel.Password)] public string Password { get; set; } = string.Empty; [FieldDefinition(4, Label = "Use Token Authentication", Type = FieldType.Checkbox, HelpText = "Use secure token-based authentication (API 1.13.0+). Disable for older servers.", Advanced = true)] public bool UseTokenAuth { get; set; } = true; [FieldDefinition(6, Type = FieldType.Number, Label = "Connection Retries", HelpText = "Number of times to retry failed connections", Advanced = true)] public int ConnectionRetries { get; set; } = 3; [FieldDefinition(7, Type = FieldType.Number, Label = "Max Parallel Downloads", HelpText = "Maximum number of downloads that can run simultaneously")] public int MaxParallelDownloads { get; set; } = 2; [FieldDefinition(8, Label = "Max Download Speed", Type = FieldType.Number, HelpText = "Set to 0 for unlimited speed. Limits download speed per file.", Unit = "KB/s", Advanced = true)] public int MaxDownloadSpeed { get; set; } = 0; [FieldDefinition(9, Label = "Preferred Audio Format", Type = FieldType.Select, SelectOptions = typeof(PreferredFormatEnum), HelpText = "Preferred audio format for transcoding (leave as 'Raw' for no transcoding)", Advanced = true)] public int PreferredFormat { get; set; } = (int)PreferredFormatEnum.Raw; [FieldDefinition(10, Label = "Max Bit Rate", Type = FieldType.Number, HelpText = "Maximum bit rate in kbps (0 for original quality)", Unit = "kbps", Advanced = true)] public int MaxBitRate { get; set; } [FieldDefinition(11, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for requests to SubSonic server", Advanced = true)] public int RequestTimeout { get; set; } = 60; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } /// /// Audio format options for SubSonic transcoding /// public enum PreferredFormatEnum { [FieldOption(Label = "No Transcoding", Hint = "Download audio in its original format.")] Raw = 0, [FieldOption(Label = "MP3", Hint = "Widely compatible lossy format.")] Mp3 = 1, [FieldOption(Label = "Opus", Hint = "Modern, efficient lossy codec.")] Opus = 2, [FieldOption(Label = "AAC", Hint = "High-quality lossy format with good compression, widely used.")] Aac = 3, [FieldOption(Label = "FLAC", Hint = "Lossless compression format that preserves original quality.")] Flac = 4 } } ================================================ FILE: Tubifarry/Download/Clients/TripleTriple/TripleTripleClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using System.Text.Json; using Tubifarry.Download.Base; namespace Tubifarry.Download.Clients.TripleTriple { public class TripleTripleClient : DownloadClientBase { private readonly ITripleTripleDownloadManager _downloadManager; private readonly INamingConfigService _namingService; private readonly IEnumerable _requestInterceptors; public TripleTripleClient( ITripleTripleDownloadManager downloadManager, IConfigService configService, IDiskProvider diskProvider, INamingConfigService namingConfigService, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, IEnumerable requestInterceptors, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _downloadManager = downloadManager; _requestInterceptors = requestInterceptors; _namingService = namingConfigService; } public override string Name => "T2Tunes"; public override string Protocol => nameof(AmazonMusicDownloadProtocol); public new TripleTripleProviderSettings Settings => base.Settings; public override Task Download(RemoteAlbum remoteAlbum, IIndexer indexer) => _downloadManager.Download(remoteAlbum, indexer, _namingService.GetConfig(), this); public override IEnumerable GetItems() => _downloadManager.GetItems(); public override void RemoveItem(DownloadClientItem item, bool deleteData) { if (deleteData) DeleteItemData(item); _downloadManager.RemoveItem(item); } public override DownloadClientInfo GetStatus() => new() { IsLocalhost = false, OutputRootFolders = [new OsPath(Settings.DownloadPath)] }; protected override void Test(List failures) { if (!_diskProvider.FolderExists(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path does not exist")); return; } if (!_diskProvider.FolderWritable(Settings.DownloadPath)) { failures.Add(new ValidationFailure("DownloadPath", "Download path is not writable")); return; } try { BaseHttpClient httpClient = new(Settings.BaseUrl.Trim(), _requestInterceptors, TimeSpan.FromSeconds(30)); string response = httpClient.GetStringAsync("/api/status").GetAwaiter().GetResult(); if (string.IsNullOrEmpty(response)) { failures.Add(new ValidationFailure("BaseUrl", "Cannot connect to T2Tunes instance: Empty response")); return; } JsonDocument doc = JsonDocument.Parse(response); if (!doc.RootElement.TryGetProperty("amazonMusic", out JsonElement statusElement) || statusElement.GetString()?.ToLower() != "up") { failures.Add(new ValidationFailure("BaseUrl", "T2Tunes Amazon Music service is not available")); return; } _logger.Debug("Successfully connected to T2Tunes, status: up"); } catch (Exception ex) { _logger.Error(ex, "Error connecting to T2Tunes instance"); failures.Add(new ValidationFailure("BaseUrl", $"Cannot connect to T2Tunes instance: {ex.Message}")); } } } } ================================================ FILE: Tubifarry/Download/Clients/TripleTriple/TripleTripleDownloadManager.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Tubifarry.Download.Base; using Tubifarry.Indexers.TripleTriple; namespace Tubifarry.Download.Clients.TripleTriple { public interface ITripleTripleDownloadManager : IBaseDownloadManager { } public class TripleTripleDownloadManager : BaseDownloadManager, ITripleTripleDownloadManager { private readonly IEnumerable _requestInterceptors; public TripleTripleDownloadManager(IEnumerable requestInterceptors, Logger logger) : base(logger) { _requestInterceptors = requestInterceptors; } protected override Task CreateDownloadRequest( RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, TripleTripleClient provider) { string baseUrl = provider.Settings.BaseUrl; bool isTrack = remoteAlbum.Release.DownloadUrl.StartsWith("track/"); TripleTripleDownloadOptions options = new() { Handler = _requesthandler, DownloadPath = provider.Settings.DownloadPath, BaseUrl = baseUrl, MaxDownloadSpeed = provider.Settings.MaxDownloadSpeed * 1024, ConnectionRetries = provider.Settings.ConnectionRetries, NamingConfig = namingConfig, RequestInterceptors = _requestInterceptors, DelayBetweenAttemps = TimeSpan.FromSeconds(2), NumberOfAttempts = (byte)provider.Settings.ConnectionRetries, ClientInfo = DownloadClientItemClientInfo.FromDownloadClient(provider, false), IsTrack = isTrack, ItemId = remoteAlbum.Release.DownloadUrl, CountryCode = ((TripleTripleCountry)provider.Settings.CountryCode).ToString(), Codec = (TripleTripleCodec)provider.Settings.Codec, DownloadLyrics = provider.Settings.DownloadLyrics, CreateLrcFile = provider.Settings.CreateLrcFile, EmbedLyrics = provider.Settings.EmbedLyrics, CoverSize = provider.Settings.CoverSize }; _requesthandler.MaxParallelism = provider.Settings.MaxParallelDownloads; return Task.FromResult(new TripleTripleDownloadRequest(remoteAlbum, options)); } } } ================================================ FILE: Tubifarry/Download/Clients/TripleTriple/TripleTripleDownloadOptions.cs ================================================ using Tubifarry.Download.Base; using Tubifarry.Indexers.TripleTriple; namespace Tubifarry.Download.Clients.TripleTriple { public record TripleTripleDownloadOptions : BaseDownloadOptions { public string CountryCode { get; set; } = "US"; public TripleTripleCodec Codec { get; set; } = TripleTripleCodec.FLAC; public bool DownloadLyrics { get; set; } = true; public bool CreateLrcFile { get; set; } = true; public bool EmbedLyrics { get; set; } = false; public int CoverSize { get; set; } = 1200; public TripleTripleDownloadOptions() : base() { } protected TripleTripleDownloadOptions(TripleTripleDownloadOptions options) : base(options) { CountryCode = options.CountryCode; Codec = options.Codec; DownloadLyrics = options.DownloadLyrics; CreateLrcFile = options.CreateLrcFile; EmbedLyrics = options.EmbedLyrics; CoverSize = options.CoverSize; } } } ================================================ FILE: Tubifarry/Download/Clients/TripleTriple/TripleTripleDownloadRequest.cs ================================================ using DownloadAssistant.Options; using DownloadAssistant.Requests; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using Tubifarry.Download.Base; using Tubifarry.Indexers.TripleTriple; namespace Tubifarry.Download.Clients.TripleTriple { public class TripleTripleDownloadRequest : BaseDownloadRequest { private readonly BaseHttpClient _httpClient; private TripleTripleAlbumInfo? _currentAlbum; private List? _mediaResponses; public TripleTripleDownloadRequest(RemoteAlbum remoteAlbum, TripleTripleDownloadOptions? options) : base(remoteAlbum, options) { _httpClient = new BaseHttpClient(Options.BaseUrl, Options.RequestInterceptors, TimeSpan.FromSeconds(Options.RequestTimeout)); _requestContainer.Add(new OwnRequest(async (token) => { try { await ProcessDownloadAsync(token); return true; } catch (Exception ex) { LogAndAppendMessage($"Error processing download: {ex.Message}", LogLevel.Error); throw; } }, new RequestOptions() { CancellationToken = Token, DelayBetweenAttemps = Options.DelayBetweenAttemps, NumberOfAttempts = Options.NumberOfAttempts, Priority = RequestPriority.Low, Handler = Options.Handler })); } protected override async Task ProcessDownloadAsync(CancellationToken token) { _logger.Trace($"Processing {(Options.IsTrack ? "track" : "album")}: {ReleaseInfo.Title}"); string asin = Options.ItemId.Split('/').Last(); if (Options.IsTrack) await ProcessSingleTrackAsync(asin, token); else await ProcessAlbumAsync(asin, token); } private async Task ProcessSingleTrackAsync(string asin, CancellationToken token) { _logger.Trace($"Processing single track with ASIN: {asin}"); TripleTripleMediaResponse? media = await GetMediaAsync(asin, token); if (media == null || !media.Streamable || media.StreamInfo == null) throw new Exception("Failed to get stream info for track"); _mediaResponses = [media]; string coverUrl = BuildCoverUrl(media.TemplateCoverUrl); await DownloadAlbumCoverAsync(coverUrl, token); Track trackMetadata = CreateTrackFromMedia(media); Album albumMetadata = CreateAlbumFromMedia(media); string extension = AudioFormatHelper.GetFileExtensionForFormat(AudioFormatHelper.GetAudioFormatFromCodec(media.StreamInfo.Codec)); string fileName = BuildTrackFilename(trackMetadata, albumMetadata, extension); InitiateDownload(media, fileName, token); _requestContainer.Add(_trackContainer); } private async Task ProcessAlbumAsync(string asin, CancellationToken token) { _logger.Trace($"Processing album with ASIN: {asin}"); _currentAlbum = await GetAlbumMetadataAsync(asin, token); if (_currentAlbum == null || (_currentAlbum.Tracks?.Count ?? 0) == 0) throw new Exception("No tracks found in album"); _expectedTrackCount = _currentAlbum.Tracks!.Count; _logger.Trace($"Found {_currentAlbum.Tracks.Count} tracks in album: {_currentAlbum.Title}"); string coverUrl = BuildCoverUrl(_currentAlbum.Image); await DownloadAlbumCoverAsync(coverUrl, token); _mediaResponses = await GetAlbumMediaAsync(asin, token); if (_mediaResponses == null || _mediaResponses.Count == 0) throw new Exception("Failed to get media info for album"); foreach (TripleTripleTrackInfo track in _currentAlbum.Tracks) { try { TripleTripleMediaResponse? media = _mediaResponses.FirstOrDefault(m => m.Asin == track.Asin); if (media == null || !media.Streamable || media.StreamInfo == null) { string reason = media == null ? "not found in media response" : !media.Streamable ? "not streamable in your region" : "missing stream info"; _logger.Debug($"Skipping track '{track.Title}': {reason}"); continue; } Track trackMetadata = CreateTrackFromAlbum(track, _currentAlbum); Album albumMetadata = CreateAlbumFromMetadata(_currentAlbum); string extension = AudioFormatHelper.GetFileExtensionForFormat(AudioFormatHelper.GetAudioFormatFromCodec(media.StreamInfo.Codec)); string trackFileName = BuildTrackFilename(trackMetadata, albumMetadata, extension); InitiateDownload(media, trackFileName, token); _logger.Trace($"Track queued: {track.Title}"); } catch (Exception ex) { LogAndAppendMessage($"Track failed: {track.Title} - {ex.Message}", LogLevel.Error); } } _requestContainer.Add(_trackContainer); } private async Task GetMediaAsync(string asin, CancellationToken token) { try { string codec = Options.Codec.ToString().ToLowerInvariant(); string url = $"/api/amazon-music/media-from-asin?asin={asin}&country={Options.CountryCode}&codec={codec}"; string response = await RequestAsync(url, token); List? mediaList = JsonSerializer.Deserialize>(response, IndexerParserHelper.StandardJsonOptions); return mediaList?.FirstOrDefault(); } catch (Exception ex) { throw new Exception($"Failed to get media info: {ex.Message}", ex); } } private async Task GetAlbumMetadataAsync(string asin, CancellationToken token) { try { string url = $"/api/amazon-music/metadata?asin={asin}&country={Options.CountryCode}"; string response = await RequestAsync(url, token); TripleTripleAlbumMetadata? metadata = JsonSerializer.Deserialize(response, IndexerParserHelper.StandardJsonOptions); return metadata?.AlbumList?.FirstOrDefault(); } catch (Exception ex) { throw new Exception($"Failed to get album metadata: {ex.Message}", ex); } } private async Task?> GetAlbumMediaAsync(string asin, CancellationToken token) { try { string codec = Options.Codec.ToString().ToLowerInvariant(); string url = $"/api/amazon-music/media-from-asin?asin={asin}&country={Options.CountryCode}&codec={codec}"; string response = await RequestAsync(url, token); return JsonSerializer.Deserialize>(response, IndexerParserHelper.StandardJsonOptions); } catch (Exception ex) { throw new Exception($"Failed to get album media: {ex.Message}", ex); } } private async Task RequestAsync(string url, CancellationToken token) { using HttpRequestMessage request = _httpClient.CreateRequest(HttpMethod.Get, url); request.Headers.Add("Referer", Options.BaseUrl); using HttpResponseMessage response = await _httpClient.SendAsync(request, token); response.EnsureSuccessStatusCode(); return await response.Content.ReadAsStringAsync(token); } private async Task DownloadAlbumCoverAsync(string? coverUrl, CancellationToken token) { if (string.IsNullOrEmpty(coverUrl)) return; try { using HttpResponseMessage response = await _httpClient.GetAsync(coverUrl, token); response.EnsureSuccessStatusCode(); _albumCover = await response.Content.ReadAsByteArrayAsync(token); _logger.Trace($"Downloaded album cover: {_albumCover.Length} bytes"); } catch (Exception ex) { _logger.Warn(ex, "Failed to download album cover"); _albumCover = null; } } private string BuildCoverUrl(string? template) { if (string.IsNullOrEmpty(template)) return string.Empty; return template .Replace("{size}", Options.CoverSize.ToString()) .Replace("{jpegQuality}", "90") .Replace("{format}", "jpg"); } private void InitiateDownload(TripleTripleMediaResponse media, string fileName, CancellationToken token) { LoadRequest downloadRequest = new(media.StreamInfo!.StreamUrl, new LoadRequestOptions() { CancellationToken = token, CreateSpeedReporter = true, SpeedReporterTimeout = 1000, Priority = RequestPriority.Normal, MaxBytesPerSecond = Options.MaxDownloadSpeed, DelayBetweenAttemps = Options.DelayBetweenAttemps, Filename = fileName, AutoStart = true, DestinationPath = _destinationPath.FullPath, Handler = Options.Handler, DeleteFilesOnFailure = true, RequestFailed = (_, __) => LogAndAppendMessage($"Download failed: {fileName}", LogLevel.Error), WriteMode = WriteMode.AppendOrTruncate, }); OwnRequest postProcessRequest = new((t) => PostProcessTrackAsync(media, downloadRequest, t), new RequestOptions() { AutoStart = false, Priority = RequestPriority.High, DelayBetweenAttemps = Options.DelayBetweenAttemps, Handler = Options.Handler, CancellationToken = token, RequestFailed = (_, __) => { LogAndAppendMessage($"Post-processing failed: {fileName}", LogLevel.Error); try { if (File.Exists(downloadRequest.Destination)) File.Delete(downloadRequest.Destination); } catch { } } }); downloadRequest.TrySetSubsequentRequest(postProcessRequest); postProcessRequest.TrySetIdle(); _trackContainer.Add(downloadRequest); _requestContainer.Add(postProcessRequest); } private async Task PostProcessTrackAsync(TripleTripleMediaResponse media, LoadRequest request, CancellationToken token) { string trackPath = request.Destination; await Task.Delay(100, token); if (!File.Exists(trackPath)) return false; try { AudioMetadataHandler audioData = new(trackPath) { AlbumCover = _albumCover }; if (!string.IsNullOrEmpty(media.DecryptionKey)) { if (!await audioData.TryDecryptAsync(media.DecryptionKey, media.StreamInfo?.Codec, token)) { _logger.Error($"Failed to decrypt track: {Path.GetFileName(trackPath)}"); return false; } } Album album = _currentAlbum != null ? CreateAlbumFromMetadata(_currentAlbum) : CreateAlbumFromMedia(media); Track track = CreateTrackFromMedia(media); if (Options.DownloadLyrics) { string? syncedLyrics = media.Lyrics?.Synced ?? media.Tags?.PlainLyrics; if (!string.IsNullOrEmpty(syncedLyrics)) audioData.Lyric = ParseSyncedLyrics(syncedLyrics); } if (!audioData.TryEmbedMetadata(album, track)) { _logger.Warn($"Failed to embed metadata for: {Path.GetFileName(audioData.TrackPath)}"); return false; } if (Options.CreateLrcFile && audioData.Lyric != null) await audioData.TryCreateLrcFileAsync(token); _logger.Trace($"Successfully processed track: {Path.GetFileName(audioData.TrackPath)}"); return true; } catch (Exception ex) { LogAndAppendMessage($"Post-processing failed for {Path.GetFileName(trackPath)}: {ex.Message}", LogLevel.Error); return false; } } private static Lyric? ParseSyncedLyrics(string syncedLyrics) { if (string.IsNullOrEmpty(syncedLyrics)) return null; List lines = []; foreach (string line in syncedLyrics.Split('\n', StringSplitOptions.RemoveEmptyEntries)) { int bracketEnd = line.IndexOf(']'); if (bracketEnd > 0 && line.StartsWith('[')) { string timestamp = line[1..bracketEnd]; string text = line[(bracketEnd + 1)..].Trim(); if (!string.IsNullOrEmpty(text)) lines.Add(new SyncLine { LrcTimestamp = $"[{timestamp}]", Line = text }); } } return lines.Count > 0 ? new Lyric(null, lines) : null; } private Album CreateAlbumFromMedia(TripleTripleMediaResponse media) => new() { Title = media.Tags?.Album ?? ReleaseInfo.Album ?? "Unknown Album", ReleaseDate = DateTime.TryParse(media.Tags?.Date, out DateTime date) ? date : ReleaseInfo.PublishDate, Artist = new LazyLoaded(new Artist { Name = media.Tags?.AlbumArtist ?? media.Tags?.Artist ?? ReleaseInfo.Artist ?? "Unknown Artist" }), AlbumReleases = new LazyLoaded>([ new() { TrackCount = media.Tags?.TrackTotal ?? 1, Title = media.Tags?.Album ?? "Unknown Album", Label = !string.IsNullOrEmpty(media.Tags?.Label) ? [media.Tags.Label] : [], } ]), Genres = !string.IsNullOrEmpty(media.Tags?.Genre) ? [media.Tags.Genre] : [] }; private Album CreateAlbumFromMetadata(TripleTripleAlbumInfo albumInfo) => new() { Title = albumInfo.Title, ReleaseDate = DateTimeOffset.FromUnixTimeMilliseconds(albumInfo.OriginalReleaseDate).DateTime, Artist = new LazyLoaded(new Artist { Name = albumInfo.Artist.Name }), AlbumReleases = new LazyLoaded>([ new() { TrackCount = albumInfo.TrackCount, Title = albumInfo.Title, Label = !string.IsNullOrEmpty(albumInfo.Label) ? [albumInfo.Label] : [], } ]), Genres = !string.IsNullOrEmpty(albumInfo.PrimaryGenreName) ? [albumInfo.PrimaryGenreName] : [] }; private Track CreateTrackFromMedia(TripleTripleMediaResponse media) => new() { Title = media.Tags?.Title ?? "Unknown Track", TrackNumber = media.Tags?.Track.ToString() ?? "1", AbsoluteTrackNumber = media.Tags?.Track ?? 1, MediumNumber = media.Tags?.Disc ?? 1, Artist = new LazyLoaded(new Artist { Name = media.Tags?.Artist ?? "Unknown Artist" }), ForeignRecordingId = media.Tags?.Isrc }; private Track CreateTrackFromAlbum(TripleTripleTrackInfo track, TripleTripleAlbumInfo album) => new() { Title = track.Title, TrackNumber = track.TrackNum.ToString(), AbsoluteTrackNumber = track.TrackNum, MediumNumber = 1, Duration = track.Duration * 1000, Artist = new LazyLoaded(new Artist { Name = album.Artist.Name }), ForeignRecordingId = track.Isrc }; } } ================================================ FILE: Tubifarry/Download/Clients/TripleTriple/TripleTripleProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; using Tubifarry.Indexers.TripleTriple; namespace Tubifarry.Download.Clients.TripleTriple { public class TripleTripleProviderSettingsValidator : AbstractValidator { public TripleTripleProviderSettingsValidator() { RuleFor(x => x.DownloadPath) .IsValidPath() .WithMessage("Download path must be a valid directory."); RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); RuleFor(x => x.ConnectionRetries) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(10) .WithMessage("Connection retries must be between 1 and 10."); RuleFor(x => x.MaxParallelDownloads) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(5) .WithMessage("Max parallel downloads must be between 1 and 5."); RuleFor(x => x.MaxDownloadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Max download speed must be greater than or equal to 0.") .LessThanOrEqualTo(100_000) .WithMessage("Max download speed must be less than or equal to 100 MB/s."); RuleFor(x => x.CoverSize) .InclusiveBetween(100, 2000) .WithMessage("Cover size must be between 100 and 2000 pixels."); } } public class TripleTripleProviderSettings : IProviderConfig { private static readonly TripleTripleProviderSettingsValidator Validator = new(); public TripleTripleProviderSettings() { ConnectionRetries = 3; MaxParallelDownloads = 2; MaxDownloadSpeed = 0; CoverSize = 1200; DownloadLyrics = true; CreateLrcFile = true; EmbedLyrics = false; } [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Directory where downloaded files will be saved")] public string DownloadPath { get; set; } = string.Empty; [FieldDefinition(1, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the T2Tunes API instance", Placeholder = "https://T2Tunes.site")] public string BaseUrl { get; set; } = string.Empty; [FieldDefinition(2, Label = "Country", Type = FieldType.Select, SelectOptions = typeof(TripleTripleCountry), HelpText = "Country code for Amazon Music region")] public int CountryCode { get; set; } = (int)TripleTripleCountry.US; [FieldDefinition(3, Label = "Preferred Codec", Type = FieldType.Select, SelectOptions = typeof(TripleTripleCodec), HelpText = "Audio codec preference for downloads")] public int Codec { get; set; } = (int)TripleTripleCodec.FLAC; [FieldDefinition(4, Label = "Download Lyrics", Type = FieldType.Checkbox, HelpText = "Download lyrics when available")] public bool DownloadLyrics { get; set; } [FieldDefinition(5, Label = "Create LRC File", Type = FieldType.Checkbox, HelpText = "Create .lrc file with synced lyrics alongside audio file")] public bool CreateLrcFile { get; set; } [FieldDefinition(6, Label = "Embed Lyrics in Audio", Type = FieldType.Checkbox, HelpText = "Embed lyrics directly in audio file metadata")] public bool EmbedLyrics { get; set; } [FieldDefinition(7, Label = "Cover Size", Type = FieldType.Number, HelpText = "Album cover image size in pixels", Unit = "px", Advanced = true)] public int CoverSize { get; set; } [FieldDefinition(8, Type = FieldType.Number, Label = "Connection Retries", HelpText = "Number of times to retry failed connections", Advanced = true)] public int ConnectionRetries { get; set; } [FieldDefinition(9, Type = FieldType.Number, Label = "Max Parallel Downloads", HelpText = "Maximum number of downloads that can run simultaneously")] public int MaxParallelDownloads { get; set; } [FieldDefinition(10, Label = "Max Download Speed", Type = FieldType.Number, HelpText = "Set to 0 for unlimited speed. Limits download speed per file.", Unit = "KB/s", Advanced = true)] public int MaxDownloadSpeed { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/SponsorBlock.cs ================================================ using NLog; using System.Globalization; using System.Net; using System.Text.Json; using System.Text.Json.Serialization; using Xabe.FFmpeg; namespace Tubifarry.Download.Clients.YouTube { public class SponsorBlock { private readonly Logger _logger = LogManager.GetCurrentClassLogger(); private readonly string _filePath; private readonly string _videoId; private readonly string _apiEndpoint; public SponsorBlock(string filePath, string videoId, string apiEndpoint = "https://sponsor.ajay.app") { if (string.IsNullOrWhiteSpace(filePath)) throw new ArgumentException("File path cannot be null or empty"); if (string.IsNullOrWhiteSpace(videoId)) throw new ArgumentException("Video ID cannot be null or empty"); if (videoId.Length != 11) throw new ArgumentException("Video ID must be exactly 11 characters"); if (string.IsNullOrWhiteSpace(apiEndpoint)) throw new ArgumentException("API endpoint cannot be null or empty"); _filePath = filePath; _videoId = videoId; _apiEndpoint = apiEndpoint; } public async Task LookupAndTrimAsync(CancellationToken cancellationToken = default) { try { if (!File.Exists(_filePath)) { _logger.Error($"Audio file not found: {_filePath}"); return false; } List segments = await FetchNonMusicSegmentsAsync(cancellationToken); if (segments.Count == 0) { _logger.Trace($"No non-music segments found for video {_videoId}"); return true; } bool result = await TrimSegmentsAsync(segments, cancellationToken); _logger.Debug("SponsorBlock processing completed"); return result; } catch (Exception ex) { _logger.Error(ex, $"Failed to process SponsorBlock segments for {_videoId}"); return false; } } private async Task> FetchNonMusicSegmentsAsync(CancellationToken cancellationToken) { string url = $"{_apiEndpoint.TrimEnd('/')}/api/skipSegments?videoID={_videoId}&category=music_offtopic&actionType=skip"; try { using HttpClient client = new(); HttpResponseMessage response = await client.GetAsync(url, cancellationToken); if (response.StatusCode == HttpStatusCode.NotFound) return []; if (!response.IsSuccessStatusCode) { _logger.Warn($"SponsorBlock API returned {response.StatusCode} for video {_videoId}"); return []; } string jsonContent = await response.Content.ReadAsStringAsync(cancellationToken); if (string.IsNullOrWhiteSpace(jsonContent)) return []; List? segments = JsonSerializer.Deserialize>(jsonContent); if (segments == null) return []; return segments.Where(s => s.Segment?.Length == 2 && s.Segment[0] < s.Segment[1]).ToList(); } catch (Exception ex) { _logger.Error(ex, $"Failed to fetch SponsorBlock data for video {_videoId}"); return []; } } private async Task TrimSegmentsAsync(List segments, CancellationToken cancellationToken) { string inputDir = Path.GetDirectoryName(_filePath) ?? ""; string trackName = Path.GetFileNameWithoutExtension(_filePath); string tempDir = Path.Combine(inputDir, $"sponsorblock_temp_{(trackName.Length > 10 ? trackName[^10..] : trackName)}"); Directory.CreateDirectory(tempDir); try { List segmentFiles = []; List sortedSegments = [.. segments.OrderBy(s => s.Segment![0])]; double previousEnd = 0.0; int segmentIndex = 0; // Get total duration double totalDuration = sortedSegments.Max(s => s.VideoDuration > 0 ? s.VideoDuration : s.Segment![1] + 30); // Extract segments foreach (SponsorSegment? segment in sortedSegments) { double segmentStart = segment.Segment![0]; double segmentEnd = segment.Segment[1]; if (segmentStart > previousEnd) { string segmentFile = Path.Combine(tempDir, $"segment_{segmentIndex:D3}{Path.GetExtension(_filePath)}"); IConversion extraction = FFmpeg.Conversions.New() .AddParameter($"-i \"{_filePath}\"") .AddParameter($"-ss {previousEnd.ToString("F3", CultureInfo.InvariantCulture)}") .AddParameter($"-to {segmentStart.ToString("F3", CultureInfo.InvariantCulture)}") .AddParameter("-c copy") .AddParameter("-map 0") // Copy all streams .AddParameter("-avoid_negative_ts make_zero") .SetOverwriteOutput(true) .SetOutput(segmentFile); await extraction.Start(cancellationToken); segmentFiles.Add(segmentFile); segmentIndex++; } previousEnd = segmentEnd; } // Add final segment if (previousEnd < totalDuration - 1) { string segmentFile = Path.Combine(tempDir, $"segment_{segmentIndex:D3}{Path.GetExtension(_filePath)}"); IConversion extraction = FFmpeg.Conversions.New() .AddParameter($"-i \"{_filePath}\"") .AddParameter($"-ss {previousEnd.ToString("F3", CultureInfo.InvariantCulture)}") .AddParameter("-c copy") .AddParameter("-map 0") // Copy all streams .SetOverwriteOutput(true) .SetOutput(segmentFile); await extraction.Start(cancellationToken); segmentFiles.Add(segmentFile); } if (segmentFiles.Count == 0) { _logger.Warn($"No segments to concatenate for video {_videoId}"); return false; } // Create concat list string concatListPath = Path.Combine(tempDir, "concat_list.txt"); await File.WriteAllLinesAsync(concatListPath, segmentFiles.Select(f => $"file '{Path.GetFileName(f)}'"), cancellationToken); // Concatenate with proper metadata handling string tempOutputPath = Path.Combine(inputDir, $"sponsorblock_{Guid.NewGuid()}{Path.GetExtension(_filePath)}"); IConversion concat = FFmpeg.Conversions.New() .AddParameter("-f concat") .AddParameter("-safe 0") .AddParameter($"-i \"{concatListPath}\"") .AddParameter($"-i \"{_filePath}\"") // Add original file for metadata .AddParameter("-c copy") .AddParameter("-map 0") // Map concat result .AddParameter("-map_metadata 1") // Take metadata from original file .SetOverwriteOutput(true) .SetOutput(tempOutputPath); await concat.Start(cancellationToken); if (File.Exists(tempOutputPath) && new FileInfo(tempOutputPath).Length > 0) { File.Move(tempOutputPath, _filePath, overwrite: true); _logger.Debug($"Successfully removed {sortedSegments.Count} non-music segments from {Path.GetFileName(_filePath)}"); return true; } _logger.Error($"Concatenation failed for video {_videoId}"); return false; } finally { try { if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); } catch (Exception ex) { _logger.Warn(ex, $"Failed to cleanup temp directory: {tempDir}"); } } } private record SponsorSegment( [property: JsonPropertyName("segment")] double[]? Segment, [property: JsonPropertyName("UUID")] string? UUID, [property: JsonPropertyName("category")] string? Category, [property: JsonPropertyName("videoDuration")] double VideoDuration, [property: JsonPropertyName("actionType")] string? ActionType, [property: JsonPropertyName("locked")] int Locked, [property: JsonPropertyName("votes")] int Votes, [property: JsonPropertyName("description")] string? Description ); } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/TrustedSessionHelper.cs ================================================ using DownloadAssistant.Base; using NLog; using NzbDrone.Common.Instrumentation; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using YouTubeMusicAPI.Client; using YouTubeSessionGenerator; using YouTubeSessionGenerator.BotGuard; using YouTubeSessionGenerator.Js.Environments; namespace Tubifarry.Download.Clients.YouTube { /// /// A centralized helper for managing YouTube trusted session authentication /// public class TrustedSessionHelper { private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(TrustedSessionHelper)); private static SessionTokens? _cachedTokens; private static string? _cachedVisitorData; private static DateTime _visitorDataExpiry = DateTime.MinValue; private static readonly SemaphoreSlim _semaphore = new(1, 1); private static readonly SemaphoreSlim _visitorDataSemaphore = new(1, 1); private static readonly Lazy NodeJsAvailable = new(() => { try { _logger.Trace("Checking Node.js availability..."); using NodeEnvironment? testEnv = new(); _logger.Debug("Node.js environment is available for local token generation"); return true; } catch (Exception ex) { _logger.Trace(ex, "Node.js environment is not available for local token generation: {Message}", ex.Message); return false; } }, LazyThreadSafetyMode.ExecutionAndPublication); /// /// Gets trusted session tokens (session-level), using cache if available and valid /// public static async Task GetTrustedSessionTokensAsync(string? serviceUrl = null, bool forceRefresh = false, CancellationToken token = default) { try { await _semaphore.WaitAsync(token); if (!forceRefresh && _cachedTokens?.IsValid == true) { _logger.Trace($"Using cached trusted session tokens from {_cachedTokens.Source}, expires in {_cachedTokens.TimeUntilExpiry:hh\\:mm\\:ss}"); return _cachedTokens; } SessionTokens newTokens = new("", "", DateTime.UtcNow.AddHours(12)); if (!string.IsNullOrEmpty(serviceUrl)) { _logger.Trace($"Using web service approach with URL: {serviceUrl}"); newTokens = await GetTokensFromWebServiceAsync(serviceUrl, null, token); } else if (IsNodeJsAvailable()) { _logger.Trace("Using local YouTubeSessionGenerator"); newTokens = await GetTokensFromLocalGeneratorAsync(null, token); } _cachedTokens = newTokens; return newTokens; } catch (HttpRequestException ex) { _logger.Error(ex, $"HTTP request to trusted session generator failed: {ex.Message}"); throw; } catch (JsonException ex) { throw new TrustedSessionException("Failed to parse JSON response from trusted session generator", ex); } catch (Exception ex) when (ex is not TrustedSessionException) { throw new TrustedSessionException($"Unexpected error fetching trusted session tokens: {ex.Message}", ex); } finally { _semaphore.Release(); } } /// /// Gets video-bound tokens for a specific video ID (not cached) /// public static async Task GetVideoBoundTokensAsync(string videoId, string? serviceUrl = null, CancellationToken token = default) { try { if (!string.IsNullOrEmpty(serviceUrl)) { _logger.Trace($"Generating video-bound token for {videoId} using bgutil service"); return await GetTokensFromWebServiceAsync(serviceUrl, videoId, token); } else if (IsNodeJsAvailable()) { _logger.Trace($"Generating video-bound token for {videoId} using local generator"); return await GetTokensFromLocalGeneratorAsync(videoId, token); } else { _logger.Warn("No token generation method available. Returning empty tokens."); return new SessionTokens("", "", DateTime.UtcNow.AddMinutes(30)); } } catch (Exception ex) { _logger.Error(ex, $"Failed to generate video-bound token for {videoId}"); throw new TrustedSessionException($"Failed to generate video-bound token for {videoId}: {ex.Message}", ex); } } /// /// Creates session information based on the provided authentication configuration /// public static async Task CreateSessionInfoAsync(string? trustedSessionGeneratorUrl = null, string? cookiePath = null, bool forceRefresh = false) { SessionTokens? effectiveTokens = null; Cookie[]? cookies = null; try { effectiveTokens = await GetTrustedSessionTokensAsync(trustedSessionGeneratorUrl, forceRefresh); _logger.Trace($"Successfully retrieved tokens from {effectiveTokens.Source}"); } catch (Exception ex) { _logger.Error(ex, "Failed to retrieve tokens for session"); } if (!string.IsNullOrEmpty(cookiePath)) cookies = LoadCookies(cookiePath); return new ClientSessionInfo(effectiveTokens, cookies); } /// /// Creates an authenticated YouTube Music client from session information /// public static YouTubeMusicClient CreateAuthenticatedClient(ClientSessionInfo sessionInfo) { HttpClientHandler handler = new() { UseCookies = false }; HttpClient httpClient = new(handler); YouTubeMusicClient client = new( // logger: new YouTubeSessionGeneratorLogger(_logger), geographicalLocation: sessionInfo.GeographicalLocation, visitorData: sessionInfo.Tokens?.VisitorData, poToken: sessionInfo.Tokens?.PoToken, cookies: sessionInfo.Cookies, httpClient: httpClient ); _logger.Debug($"Created YouTube client with: {sessionInfo.AuthenticationSummary}"); return client; } /// /// Creates an authenticated YouTube Music client with the specified configuration /// public static async Task CreateAuthenticatedClientAsync(string? trustedSessionGeneratorUrl = null, string? cookiePath = null, bool forceRefresh = false) { ClientSessionInfo sessionInfo = await CreateSessionInfoAsync(trustedSessionGeneratorUrl, cookiePath, forceRefresh); return CreateAuthenticatedClient(sessionInfo); } /// /// Updates an existing YouTube Music client with video-bound tokens /// public static async Task UpdateClientWithVideoBoundTokensAsync(YouTubeMusicClient client, string videoId, string? serviceUrl = null, CancellationToken token = default) { SessionTokens videoBoundTokens = await GetVideoBoundTokensAsync(videoId, serviceUrl, token); client.PoToken = videoBoundTokens.PoToken; client.VisitorData = videoBoundTokens.VisitorData; _logger.Trace($"Updated client with video-bound tokens for {videoId}"); } public static Cookie[]? LoadCookies(string cookiePath) { _logger?.Debug($"Trying to parse cookies from {cookiePath}"); try { if (File.Exists(cookiePath)) { Cookie[] cookies = CookieManager.ParseCookieFile(cookiePath); if (cookies?.Length > 0) { _logger?.Trace($"Successfully parsed {cookies.Length} cookies"); return cookies; } } else { _logger?.Warn($"Cookie file not found: {cookiePath}"); } } catch (Exception ex) { _logger?.Error(ex, $"Failed to parse cookies from {cookiePath}"); } return null; } /// /// Fetches tokens from a Web Service /// private static async Task GetTokensFromWebServiceAsync(string serviceUrl, string? videoId, CancellationToken token) { if (string.IsNullOrEmpty(serviceUrl)) throw new ArgumentNullException(nameof(serviceUrl), "Service URL cannot be null or empty"); string baseUrl = serviceUrl.TrimEnd('/'); string url = baseUrl + "/get_pot"; _logger.Trace($"Fetching token from bgutil service: {url}" + (videoId != null ? $" for video {videoId}" : " (session-level)")); // Build request body Dictionary requestBody = new() { ["bypass_cache"] = false }; if (!string.IsNullOrEmpty(videoId)) { requestBody["content_binding"] = videoId; } string jsonBody = JsonSerializer.Serialize(requestBody); HttpRequestMessage request = new(HttpMethod.Post, url) { Content = new StringContent(jsonBody, Encoding.UTF8, "application/json") }; request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); using HttpResponseMessage response = await HttpGet.HttpClient.SendAsync(request, token); response.EnsureSuccessStatusCode(); string responseContent = await response.Content.ReadAsStringAsync(token); _logger.Trace($"Received response: {responseContent}"); return ParseResponse(responseContent, videoId != null ? "bgutil (Video-Bound)" : "bgutil (Session)"); } private static SessionTokens ParseResponse(string responseJson, string source) { JsonDocument jsonDoc = JsonDocument.Parse(responseJson); JsonElement root = jsonDoc.RootElement; if (!root.TryGetProperty("poToken", out JsonElement poTokenElement) || !root.TryGetProperty("contentBinding", out JsonElement contentBindingElement) || !root.TryGetProperty("expiresAt", out JsonElement expiresAtElement)) { throw new TrustedSessionException($"Invalid response format from bgutil service: {responseJson}"); } string? poToken = poTokenElement.GetString(); string? contentBinding = contentBindingElement.GetString(); string? expiresAtStr = expiresAtElement.GetString(); if (string.IsNullOrEmpty(poToken) || string.IsNullOrEmpty(contentBinding)) throw new TrustedSessionException("Received empty token values from bgutil service"); DateTime expiryDateTime = DateTimeOffset.Parse(expiresAtStr!).UtcDateTime; SessionTokens sessionTokens = new(poToken, contentBinding, expiryDateTime, source); _logger.Trace($"Successfully fetched tokens from {source}. Expiry: {expiryDateTime}"); return sessionTokens; } /// /// Gets or creates cached visitor data (prevents rate limiting from YouTube) /// private static async Task GetOrCreateVisitorDataAsync(CancellationToken token) { await _visitorDataSemaphore.WaitAsync(token); try { if (!string.IsNullOrEmpty(_cachedVisitorData) && DateTime.UtcNow < _visitorDataExpiry) { _logger.Trace($"Using cached visitor data, expires in {(_visitorDataExpiry - DateTime.UtcNow):hh\\:mm\\:ss}"); return _cachedVisitorData; } _logger.Trace("Generating fresh visitor data from YouTube (only once per session)..."); using NodeEnvironment tempEnv = new NodeEnvironment(); YouTubeSessionConfig config = new() { JsEnvironment = tempEnv, HttpClient = HttpGet.HttpClient, }; YouTubeSessionCreator tempCreator = new(config); _cachedVisitorData = await tempCreator.VisitorDataAsync(token); _visitorDataExpiry = DateTime.UtcNow.AddHours(4); _logger.Debug($"Successfully generated visitor data, expires at {_visitorDataExpiry}"); return _cachedVisitorData; } finally { _visitorDataSemaphore.Release(); } } /// /// Generates tokens using local YouTubeSessionGenerator /// private static async Task GetTokensFromLocalGeneratorAsync(string? videoId, CancellationToken token) { NodeEnvironment? nodeEnvironment = null; try { string visitorData = await GetOrCreateVisitorDataAsync(token); _logger.Trace("Initializing Node.js environment for token generation"); nodeEnvironment = new NodeEnvironment(); YouTubeSessionConfig config = new() { JsEnvironment = nodeEnvironment, HttpClient = HttpGet.HttpClient, }; YouTubeSessionCreator creator = new(config); BotGuardContentBinding? contentBinding = null; string tokenType = "Session"; // If videoId is provided, create video-bound token if (!string.IsNullOrEmpty(videoId)) { _logger.Trace($"Creating content binding for video: {videoId}"); contentBinding = new BotGuardContentBinding { EncryptedVideoId = videoId }; tokenType = "Video-Bound"; } _logger.Trace($"Generating {tokenType} proof of origin token..."); string mintIdentifier = !string.IsNullOrEmpty(videoId) ? videoId : visitorData; string poToken = await creator.ProofOfOriginTokenAsync(mintIdentifier, contentBinding, token); DateTime expiryTime = DateTime.UtcNow.AddHours(4); SessionTokens sessionTokens = new(poToken, visitorData, expiryTime, $"Local Generator ({tokenType})"); _logger.Debug($"Successfully generated {tokenType} tokens locally. Expiry: {expiryTime}"); return sessionTokens; } catch (Exception ex) { _logger.Error(ex, "Failed to generate tokens using local YouTubeSessionGenerator"); throw new TrustedSessionException("Failed to generate tokens using local generator", ex); } finally { nodeEnvironment?.Dispose(); } } /// /// Validates authentication settings and connectivity /// public static async Task ValidateAuthenticationSettingsAsync(string? trustedSessionGeneratorUrl = null, string? cookiePath = null) { if (string.IsNullOrEmpty(trustedSessionGeneratorUrl) && !IsNodeJsAvailable()) _logger.Warn("Node.js environment is not available for local token generation."); if (!string.IsNullOrEmpty(trustedSessionGeneratorUrl)) { if (!Uri.TryCreate(trustedSessionGeneratorUrl, UriKind.Absolute, out _)) throw new ArgumentException($"Invalid trusted session generator URL: {trustedSessionGeneratorUrl}"); try { string testUrl = $"{trustedSessionGeneratorUrl.TrimEnd('/')}/get_pot"; string jsonBody = JsonSerializer.Serialize(new { bypass_cache = false }); HttpRequestMessage request = new(HttpMethod.Post, testUrl) { Content = new StringContent(jsonBody, Encoding.UTF8, "application/json") }; request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); using HttpResponseMessage response = await HttpGet.HttpClient.SendAsync(request); response.EnsureSuccessStatusCode(); _logger.Trace("Successfully connected to trusted session generator"); } catch (Exception ex) { throw new ArgumentException($"Error connecting to the trusted session generator: {ex.Message}"); } } // Validate cookie path if provided if (!string.IsNullOrEmpty(cookiePath)) { if (!File.Exists(cookiePath)) throw new FileNotFoundException("Cookie file not found", cookiePath); try { Cookie[]? cookies = CookieManager.ParseCookieFile(cookiePath); if (cookies == null || cookies.Length == 0) throw new ArgumentException("No valid cookies found in the cookie file"); } catch (Exception ex) when (ex is not ArgumentException && ex is not FileNotFoundException) { throw new ArgumentException($"Error parsing cookie file: {ex.Message}"); } } } /// /// Checks if Node.js is available for local token generation /// private static bool IsNodeJsAvailable() => NodeJsAvailable.Value; public static void ClearCache() { _cachedTokens = null; _cachedVisitorData = null; _visitorDataExpiry = DateTime.MinValue; _logger.Trace("Token and visitor data cache cleared"); } public static SessionTokens? GetCachedTokens() => _cachedTokens; /// /// Logger adapter to bridge NLog with Microsoft.Extensions.Logging for YouTubeSessionGenerator /// private class YouTubeSessionGeneratorLogger : Microsoft.Extensions.Logging.ILogger { private readonly Logger _nlogLogger; public YouTubeSessionGeneratorLogger(Logger nlogLogger) => _nlogLogger = nlogLogger; public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; public void Log(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId, TState state, Exception? exception, Func formatter) { if (!IsEnabled(logLevel)) return; string message = formatter(state, exception); LogLevel nlogLevel = logLevel switch { Microsoft.Extensions.Logging.LogLevel.Trace => LogLevel.Trace, Microsoft.Extensions.Logging.LogLevel.Debug => LogLevel.Trace, Microsoft.Extensions.Logging.LogLevel.Information => LogLevel.Trace, Microsoft.Extensions.Logging.LogLevel.Warning => LogLevel.Warn, Microsoft.Extensions.Logging.LogLevel.Error => LogLevel.Error, Microsoft.Extensions.Logging.LogLevel.Critical => LogLevel.Fatal, _ => LogLevel.Info }; _nlogLogger.Log(nlogLevel, exception, message); } } } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/YouTubeDownloadOptions.cs ================================================ using Tubifarry.Download.Base; using YouTubeMusicAPI.Client; namespace Tubifarry.Download.Clients.YouTube { /// /// YouTube-specific download options extending the base options /// public record YouTubeDownloadOptions : BaseDownloadOptions { /// /// YouTube Music API client instance /// public YouTubeMusicClient? YouTubeMusicClient { get; set; } /// /// Re-encoding options for audio format conversion /// public ReEncodeOptions ReEncodeOptions { get; set; } /// /// Whether to use ID3v2.3 tags instead of ID3v2.4 /// public bool UseID3v2_3 { get; set; } /// /// Minimum random delay in milliseconds /// public int RandomDelayMin { get; set; } = 100; /// /// Maximum random delay in milliseconds /// public int RandomDelayMax { get; set; } = 2000; /// /// Whether to use SponsorBlock for trimming /// public bool UseSponsorBlock { get; set; } /// /// SponsorBlock API endpoint URL /// public string SponsorBlockApiEndpoint { get; set; } = "https://sponsor.ajay.app"; /// /// URL to the Trusted Session Generator service /// public string? TrustedSessionGeneratorUrl { get; set; } public YouTubeDownloadOptions() { } protected YouTubeDownloadOptions(YouTubeDownloadOptions options) : base(options) { YouTubeMusicClient = options.YouTubeMusicClient; ReEncodeOptions = options.ReEncodeOptions; UseID3v2_3 = options.UseID3v2_3; RandomDelayMin = options.RandomDelayMin; RandomDelayMax = options.RandomDelayMax; UseSponsorBlock = options.UseSponsorBlock; SponsorBlockApiEndpoint = options.SponsorBlockApiEndpoint; TrustedSessionGeneratorUrl = options.TrustedSessionGeneratorUrl; } } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/YouTubeDownloadRequest.cs ================================================ using DownloadAssistant.Base; using DownloadAssistant.Options; using DownloadAssistant.Requests; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using Requests; using Requests.Options; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; using Tubifarry.Download.Base; using YouTubeMusicAPI.Models; using YouTubeMusicAPI.Models.Info; using YouTubeMusicAPI.Models.Streaming; namespace Tubifarry.Download.Clients.YouTube { /// /// YouTube download request handling album downloads. /// public class YouTubeDownloadRequest : BaseDownloadRequest { public YouTubeDownloadRequest(RemoteAlbum remoteAlbum, YouTubeDownloadOptions? options) : base(remoteAlbum, options) { Options.YouTubeMusicClient ??= TrustedSessionHelper.CreateAuthenticatedClientAsync().GetAwaiter().GetResult(); _requestContainer.Add(new OwnRequest(async (token) => { try { await ProcessDownloadAsync(token); return true; } catch (Exception ex) { LogAndAppendMessage($"Error processing download: {ex.Message}", LogLevel.Error); throw; } }, new RequestOptions() { CancellationToken = Token, DelayBetweenAttemps = Options.DelayBetweenAttemps, NumberOfAttempts = Options.NumberOfAttempts, Priority = RequestPriority.Low, Handler = Options.Handler })); } protected override async Task ProcessDownloadAsync(CancellationToken token) { _logger.Trace($"Processing YouTube album: {ReleaseInfo.Title}"); await ProcessAlbumAsync(Options.ItemId, token); } private async Task ProcessAlbumAsync(string downloadUrl, CancellationToken token) { string albumBrowseID = await Options.YouTubeMusicClient!.GetAlbumBrowseIdAsync(downloadUrl, token).ConfigureAwait(false); AlbumInfo albumInfo = await Options.YouTubeMusicClient.GetAlbumInfoAsync(albumBrowseID, token).ConfigureAwait(false); await ApplyRandomDelayAsync(token); if (albumInfo?.Songs == null || albumInfo.Songs.Length == 0) { LogAndAppendMessage($"No tracks to download found in the album: {ReleaseInfo.Album}", LogLevel.Debug); return; } _expectedTrackCount = albumInfo.Songs.Length; _albumCover = await TryDownloadCoverAsync(albumInfo, token).ConfigureAwait(false); foreach (AlbumSong trackInfo in albumInfo.Songs) { if (trackInfo.Id == null) { LogAndAppendMessage($"Skipping track '{trackInfo.Name}' in album '{ReleaseInfo.Album}' because it has no valid download URL.", LogLevel.Debug); continue; } try { await TryUpdateVideoBoundTokensAsync(trackInfo.Id, token); StreamingData streamingData = await Options.YouTubeMusicClient.GetStreamingDataAsync(trackInfo.Id, token).ConfigureAwait(false); AudioStreamInfo? highestAudioStreamInfo = streamingData.StreamInfo.OfType().OrderByDescending(info => info.Bitrate).FirstOrDefault(); if (highestAudioStreamInfo == null) { LogAndAppendMessage($"Skipping track '{trackInfo.Name}' in album '{ReleaseInfo.Album}' because no audio stream was found.", LogLevel.Debug); continue; } AddTrackDownloadRequest(albumInfo, trackInfo, highestAudioStreamInfo, token); await _trackContainer.Task; } catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden) { _logger?.Warn(ex, $"403 Forbidden for track '{trackInfo.Name}' in album '{ReleaseInfo.Album}'."); } catch (Exception ex) { LogAndAppendMessage($"Failed to process track '{trackInfo.Name}' in album '{ReleaseInfo.Album}'", LogLevel.Error); _logger?.Error(ex, $"Failed to process track '{trackInfo.Name}' in album '{ReleaseInfo.Album}'."); } } _requestContainer.Add(_trackContainer); } private void AddTrackDownloadRequest(AlbumInfo albumInfo, AlbumSong trackInfo, AudioStreamInfo audioStreamInfo, CancellationToken token) { _albumData.Title = albumInfo.Name; string fileName = BuildTrackFilename(CreateTrackFromYouTubeData(trackInfo, albumInfo), _albumData, ".m4a"); LoadRequest downloadRequest = new(audioStreamInfo.Url, new LoadRequestOptions() { CancellationToken = token, CreateSpeedReporter = true, SpeedReporterTimeout = 1000, Priority = RequestPriority.Normal, MaxBytesPerSecond = Options.MaxDownloadSpeed, DelayBetweenAttemps = Options.DelayBetweenAttemps, Filename = fileName, DestinationPath = _destinationPath.FullPath, Handler = Options.Handler, NumberOfAttempts = 3, DeleteFilesOnFailure = true, Chunks = Options.Chunks, RequestFailed = (_, __) => LogAndAppendMessage($"Downloading track '{trackInfo.Name}' in album '{albumInfo.Name}' failed.", LogLevel.Debug), WriteMode = WriteMode.AppendOrTruncate, AutoStart = true }); // Add post-processing OwnRequest postProcessRequest = new((t) => PostProcessTrackAsync(albumInfo, trackInfo, downloadRequest, t), new RequestOptions() { AutoStart = false, Priority = RequestPriority.High, DelayBetweenAttemps = Options.DelayBetweenAttemps, Handler = Options.Handler, RequestFailed = (_, __) => { LogAndAppendMessage($"Post-processing for track '{trackInfo.Name}' in album '{albumInfo.Name}' failed.", LogLevel.Debug); try { if (File.Exists(downloadRequest.Destination)) File.Delete(downloadRequest.Destination); } catch { } }, CancellationToken = token }); downloadRequest.TrySetSubsequentRequest(postProcessRequest); postProcessRequest.TrySetIdle(); _trackContainer.Add(downloadRequest); _requestContainer.Add(postProcessRequest); } private async Task PostProcessTrackAsync(AlbumInfo albumInfo, AlbumSong trackInfo, LoadRequest request, CancellationToken token) { string trackPath = request.Destination; await Task.Delay(100, token); if (!File.Exists(trackPath)) return false; try { AudioMetadataHandler audioData = new(trackPath) { AlbumCover = _albumCover, UseID3v2_3 = Options.UseID3v2_3 }; AudioFormat format = AudioFormatHelper.ConvertOptionToAudioFormat(Options.ReEncodeOptions); if (Options.ReEncodeOptions == ReEncodeOptions.OnlyExtract) await audioData.TryExtractAudioFromVideoAsync(); else if (format != AudioFormat.Unknown) await audioData.TryConvertToFormatAsync(format); if (Options.UseSponsorBlock && !string.IsNullOrWhiteSpace(trackInfo.Id)) await new SponsorBlock(audioData.TrackPath, trackInfo.Id, Options.SponsorBlockApiEndpoint).LookupAndTrimAsync(token); trackPath = audioData.TrackPath; Album album = CreateAlbumFromYouTubeData(albumInfo); Track track = CreateTrackFromYouTubeData(trackInfo, albumInfo); if (!audioData.TryEmbedMetadata(album, track)) { _logger.Warn($"Failed to embed metadata for: {Path.GetFileName(trackPath)}"); return false; } _logger.Trace($"Successfully processed track: {Path.GetFileName(trackPath)}"); return true; } catch (Exception ex) { LogAndAppendMessage($"Post-processing failed for {Path.GetFileName(trackPath)}: {ex.Message}", LogLevel.Error); return false; } } private Album CreateAlbumFromYouTubeData(AlbumInfo albumInfo) => new() { Title = albumInfo.Name ?? ReleaseInfo.Album, ReleaseDate = albumInfo.ReleaseYear > 0 ? new DateTime(albumInfo.ReleaseYear, 1, 1) : ReleaseInfo.PublishDate, Artist = new LazyLoaded(new Artist { Name = albumInfo.Artists?.FirstOrDefault()?.Name ?? ReleaseInfo.Artist, }), AlbumReleases = new LazyLoaded>([ new() { TrackCount = albumInfo.SongCount, Title = albumInfo.Name ?? ReleaseInfo.Album, Duration = (int)albumInfo.Duration.TotalMilliseconds } ]), Genres = _remoteAlbum.Albums.FirstOrDefault()?.Genres }; private Track CreateTrackFromYouTubeData(AlbumSong trackInfo, AlbumInfo albumInfo) => new() { Title = trackInfo.Name, AbsoluteTrackNumber = trackInfo.SongNumber ?? 0, TrackNumber = (trackInfo.SongNumber ?? 0).ToString(), Duration = (int)trackInfo.Duration.TotalMilliseconds, Explicit = trackInfo.IsExplicit, Artist = new LazyLoaded(new Artist { Name = albumInfo.Artists?.FirstOrDefault()?.Name ?? ReleaseInfo.Artist, }) }; private async Task TryDownloadCoverAsync(AlbumInfo albumInfo, CancellationToken token) { try { Thumbnail? bestThumbnail = albumInfo.Thumbnails.OrderByDescending(x => x.Height * x.Width).FirstOrDefault(); int[] releaseResolution = ReleaseInfo.Resolution.Split('x').Select(int.Parse).ToArray(); int releaseArea = releaseResolution[0] * releaseResolution[1]; int albumArea = (bestThumbnail?.Height ?? 0) * (bestThumbnail?.Width ?? 0); string coverUrl = albumArea > releaseArea ? bestThumbnail?.Url ?? ReleaseInfo.Source : ReleaseInfo.Source; using HttpResponseMessage response = await HttpGet.HttpClient.GetAsync(coverUrl, token); if (!response.IsSuccessStatusCode) { LogAndAppendMessage($"Failed to download cover art for album '{albumInfo.Name}'. Status code: {response.StatusCode}.", LogLevel.Debug); return null; } return await response.Content.ReadAsByteArrayAsync(token); } catch (Exception ex) { _logger.Warn(ex, "Failed to download album cover"); return null; } } private async Task ApplyRandomDelayAsync(CancellationToken token) { if (Options.RandomDelayMin > 0 && Options.RandomDelayMax > 0) { int delay = new Random().Next(Options.RandomDelayMin, Options.RandomDelayMax); await Task.Delay(delay, token).ConfigureAwait(false); } } private async Task TryUpdateVideoBoundTokensAsync(string videoId, CancellationToken token) { try { _logger?.Trace($"Updating client with video-bound tokens for: {videoId}"); await TrustedSessionHelper.UpdateClientWithVideoBoundTokensAsync(Options.YouTubeMusicClient!, videoId, serviceUrl: Options.TrustedSessionGeneratorUrl, token); } catch (Exception ex) { _logger?.Warn(ex, $"Failed to generate video-bound token for {videoId}, using existing session tokens"); } } } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/YoutubeClient.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Localization; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using Requests; using Tubifarry.Core.Model; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using Xabe.FFmpeg; namespace Tubifarry.Download.Clients.YouTube { public class YoutubeClient : DownloadClientBase { private readonly IYoutubeDownloadManager _dlManager; private readonly INamingConfigService _naminService; public YoutubeClient(IYoutubeDownloadManager dlManager, IConfigService configService, IDiskProvider diskProvider, INamingConfigService namingConfigService, IRemotePathMappingService remotePathMappingService, ILocalizationService localizationService, Logger logger) : base(configService, diskProvider, remotePathMappingService, localizationService, logger) { _dlManager = dlManager; _naminService = namingConfigService; RequestHandler.MainRequestHandlers[1].MaxParallelism = 1; } public override string Name => "Youtube"; public override string Protocol => nameof(YoutubeDownloadProtocol); public new YoutubeProviderSettings Settings => base.Settings; public override Task Download(RemoteAlbum remoteAlbum, IIndexer indexer) => _dlManager.Download(remoteAlbum, indexer, _naminService.GetConfig(), this); public override IEnumerable GetItems() => _dlManager.GetItems(); public override void RemoveItem(DownloadClientItem item, bool deleteData) { if (deleteData) DeleteItemData(item); _dlManager.RemoveItem(item); } public override DownloadClientInfo GetStatus() => new() { IsLocalhost = false, OutputRootFolders = [new OsPath(Settings.DownloadPath)] }; protected override void Test(List failures) { try { TrustedSessionHelper.ValidateAuthenticationSettingsAsync(Settings.TrustedSessionGeneratorUrl, Settings.CookiePath).GetAwaiter().GetResult(); SessionTokens session = TrustedSessionHelper.GetTrustedSessionTokensAsync(Settings.TrustedSessionGeneratorUrl, true).GetAwaiter().GetResult(); if (!session.IsValid && !session.IsEmpty) failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", "Failed to retrieve valid tokens from the session generator service")); } catch (Exception ex) { failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", $"Failed to valiate session generator service: {ex.Message}")); } if (string.IsNullOrEmpty(Settings.DownloadPath)) failures.AddRange(PermissionTester.TestAllPermissions(Settings.FFmpegPath, _logger)); failures.AddIfNotNull(TestFFmpeg().GetAwaiter().GetResult()); } public async Task TestFFmpeg() { if (Settings.ReEncode != (int)ReEncodeOptions.Disabled || Settings.UseSponsorBlock) { string old = FFmpeg.ExecutablesPath; FFmpeg.SetExecutablesPath(Settings.FFmpegPath); AudioMetadataHandler.ResetFFmpegInstallationCheck(); if (!AudioMetadataHandler.CheckFFmpegInstalled()) { try { await AudioMetadataHandler.InstallFFmpeg(Settings.FFmpegPath); } catch (Exception ex) { if (!string.IsNullOrEmpty(old)) FFmpeg.SetExecutablesPath(old); return new ValidationFailure("FFmpegInstallation", $"Failed to install FFmpeg: {ex.Message}"); } } } return null!; } } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/YoutubeDownloadManager.cs ================================================ using NLog; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using Tubifarry.Core.Records; using Tubifarry.Download.Base; using YouTubeMusicAPI.Client; namespace Tubifarry.Download.Clients.YouTube { /// /// Interface for YouTube download manager /// public interface IYoutubeDownloadManager : IBaseDownloadManager; /// /// YouTube download manager using the base download manager implementation /// public class YoutubeDownloadManager : BaseDownloadManager, IYoutubeDownloadManager { private YouTubeMusicClient? _youTubeClient; private SessionTokens? _sessionToken; private Task? _testTask; public YoutubeDownloadManager(Logger logger) : base(logger) { _requesthandler.MaxParallelism = 2; } protected override async Task CreateDownloadRequest( RemoteAlbum remoteAlbum, IIndexer indexer, NamingConfig namingConfig, YoutubeClient provider) { _testTask ??= provider.TestFFmpeg(); _testTask.GetAwaiter().GetResult(); await UpdateClientAsync(provider); YouTubeDownloadOptions options = new() { YouTubeMusicClient = _youTubeClient, Handler = _requesthandler, DownloadPath = provider.Settings.DownloadPath, Chunks = provider.Settings.Chunks, DelayBetweenAttemps = TimeSpan.FromSeconds(5), NumberOfAttempts = 2, RandomDelayMin = provider.Settings.RandomDelayMin, RandomDelayMax = provider.Settings.RandomDelayMax, MaxDownloadSpeed = provider.Settings.MaxDownloadSpeed * 1024, NamingConfig = namingConfig, UseID3v2_3 = provider.Settings.UseID3v2_3, ReEncodeOptions = (ReEncodeOptions)provider.Settings.ReEncode, ClientInfo = DownloadClientItemClientInfo.FromDownloadClient(provider, false), UseSponsorBlock = provider.Settings.UseSponsorBlock, SponsorBlockApiEndpoint = provider.Settings.SponsorBlockApiEndpoint, TrustedSessionGeneratorUrl = provider.Settings.TrustedSessionGeneratorUrl, IsTrack = false, ItemId = remoteAlbum.Release.DownloadUrl }; return new YouTubeDownloadRequest(remoteAlbum, options); } private async Task UpdateClientAsync(YoutubeClient provider) { if (_sessionToken?.IsValid == true) return; _sessionToken = await TrustedSessionHelper.GetTrustedSessionTokensAsync(provider.Settings.TrustedSessionGeneratorUrl); _youTubeClient = await TrustedSessionHelper.CreateAuthenticatedClientAsync(provider.Settings.TrustedSessionGeneratorUrl, provider.Settings.CookiePath); } } } ================================================ FILE: Tubifarry/Download/Clients/YouTube/YoutubeProviderSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; using Tubifarry.Core.Utilities; namespace Tubifarry.Download.Clients.YouTube { public class YoutubeProviderSettingsValidator : AbstractValidator { public YoutubeProviderSettingsValidator() { // Validate DownloadPath RuleFor(x => x.DownloadPath) .IsValidPath() .WithMessage("Download path must be a valid directory."); // Validate CookiePath (if provided) RuleFor(x => x.CookiePath) .Must(path => string.IsNullOrEmpty(path) || System.IO.File.Exists(path)) .WithMessage("Cookie file does not exist. Please provide a valid path to the cookies file.") .Must(path => string.IsNullOrEmpty(path) || CookieManager.ParseCookieFile(path).Length != 0) .WithMessage("Cookie file is invalid or contains no valid cookies."); // Validate Chunks RuleFor(x => x.Chunks) .Must(chunks => chunks > 0 && chunks < 5) .WithMessage("Chunks must be greater than 0 and smaller than 5."); // Validate FFmpegPath (if re-encoding is enabled) RuleFor(x => x.FFmpegPath) .NotEmpty() .When(x => x.ReEncode != (int)ReEncodeOptions.Disabled) .WithMessage("FFmpeg path is required when re-encoding is enabled."); RuleFor(x => x.FFmpegPath) .IsValidPath() .When(x => x.ReEncode != (int)ReEncodeOptions.Disabled) .WithMessage("Invalid FFmpeg path. Please provide a valid path to the FFmpeg binary."); // Validate Random Delay Range RuleFor(x => x.RandomDelayMin) .LessThanOrEqualTo(x => x.RandomDelayMax) .WithMessage("Minimum delay must be less than or equal to maximum delay."); RuleFor(x => x.RandomDelayMax) .GreaterThanOrEqualTo(x => x.RandomDelayMin) .WithMessage("Maximum delay must be greater than or equal to minimum delay.") .GreaterThanOrEqualTo(_ => 0) .WithMessage("Maximum delay must be greater than or equal to 0."); // Validate Max Download Speed RuleFor(x => x.MaxDownloadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Max download speed must be greater than or equal to 0.") .LessThanOrEqualTo(2_500) .WithMessage("Max download speed must be less than or equal to 20 Mbps (2,500 KB/s)."); // Validate TrustedSessionGeneratorUrl RuleFor(x => x.TrustedSessionGeneratorUrl) .Must(url => string.IsNullOrEmpty(url) || Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Trusted Session Generator URL must be a valid URL if provided."); // Validate SponsorBlock API endpoint RuleFor(x => x.SponsorBlockApiEndpoint) .NotEmpty() .When(x => x.UseSponsorBlock) .WithMessage("SponsorBlock API endpoint is required when SponsorBlock is enabled.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .When(x => x.UseSponsorBlock && !string.IsNullOrEmpty(x.SponsorBlockApiEndpoint)) .WithMessage("SponsorBlock API endpoint must be a valid URL."); } } public class YoutubeProviderSettings : IProviderConfig { private static readonly YoutubeProviderSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Specify the directory where downloaded files will be saved.")] public string DownloadPath { get; set; } = ""; [FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Hidden = HiddenType.Visible, Placeholder = "/downloads/Cookies/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but required for accessing restricted content.", Advanced = true)] public string CookiePath { get; set; } = string.Empty; [FieldDefinition(2, Label = "Use ID3v2.3 Tags", HelpText = "Enable this option to use ID3v2.3 tags for better compatibility with older media players like Windows Media Player.", Type = FieldType.Checkbox, Advanced = true)] public bool UseID3v2_3 { get; set; } [FieldDefinition(3, Label = "ReEncode", Type = FieldType.Select, SelectOptions = typeof(ReEncodeOptions), HelpText = "Specify whether to re-encode audio files and how to handle FFmpeg.", Advanced = true)] public int ReEncode { get; set; } = (int)ReEncodeOptions.Disabled; [FieldDefinition(4, Label = "FFmpeg Path", Type = FieldType.Path, Placeholder = "/downloads/FFmpeg", HelpText = "Specify the path to the FFmpeg binary. Not required if 'Disabled' is selected.", Advanced = true)] public string FFmpegPath { get; set; } = string.Empty; [FieldDefinition(5, Label = "File Chunk Count", Type = FieldType.Number, HelpText = "Number of chunks to split the download into. Each chunk is its own download. Note: Non-chunked downloads from YouTube are typically much slower.", Advanced = true)] public int Chunks { get; set; } = 2; [FieldDefinition(6, Label = "Delay Min", Type = FieldType.Number, HelpText = "Minimum random delay between requests to avoid bot notifications.", Unit = "ms", Advanced = true)] public int RandomDelayMin { get; set; } = 100; [FieldDefinition(7, Label = "Delay Max", Type = FieldType.Number, HelpText = "Maximum random delay between requests to avoid bot notifications.", Unit = "ms", Advanced = true)] public int RandomDelayMax { get; set; } = 2000; [FieldDefinition(8, Label = "Max Download Speed", Type = FieldType.Number, HelpText = "Set to 0 for unlimited speed. Limits the download speed per download.", Unit = "KB/s", Advanced = true)] public int MaxDownloadSpeed { get; set; } [FieldDefinition(9, Label = "Trusted Session Generator URL", Type = FieldType.Textbox, Placeholder = "http://localhost:8080", HelpText = "URL to the YouTube Trusted Session Generator service. When provided, PoToken and Visitor Data will be fetched automatically.", Advanced = true)] public string TrustedSessionGeneratorUrl { get; set; } = string.Empty; [FieldDefinition(10, Label = "Use SponsorBlock", Type = FieldType.Checkbox, HelpText = "Enable SponsorBlock integration to automatically remove non-music segments (intros, outros, talking) from downloaded tracks.")] public bool UseSponsorBlock { get; set; } [FieldDefinition(11, Label = "SponsorBlock API Endpoint", Type = FieldType.Textbox, Placeholder = "https://sponsor.ajay.app", HelpText = "SponsorBlock API endpoint URL. Change only if using a custom SponsorBlock instance.", Advanced = true)] public string SponsorBlockApiEndpoint { get; set; } = "https://sponsor.ajay.app"; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum ReEncodeOptions { [FieldOption(Label = "Disabled", Hint = "No re-encoding, keep original.")] Disabled, [FieldOption(Label = "Only Extract", Hint = "Extract audio, no re-encoding.")] OnlyExtract, [FieldOption(Label = "AAC", Hint = "Re-encode to AAC (VBR).")] AAC, [FieldOption(Label = "MP3", Hint = "Re-encode to MP3 (VBR).")] MP3, [FieldOption(Label = "Opus", Hint = "Re-encode to Opus (VBR).")] Opus, [FieldOption(Label = "Vorbis", Hint = "Re-encode to Vorbis (fixed 224 kbps).")] Vorbis } } ================================================ FILE: Tubifarry/ILRepack.targets ================================================  ================================================ FILE: Tubifarry/ImportLists/ArrStack/ArrMedia.cs ================================================ using System.Text.Json.Serialization; namespace Tubifarry.ImportLists.ArrStack { /// /// Represents a media item from an Arr application (Radarr/Sonarr). /// Used for deserializing API responses from Arr applications. /// internal record class ArrMedia { /// /// The title of the media item (movie title, series name, etc.) /// [JsonPropertyName("title")] public string Title { get; set; } = string.Empty; /// /// Unique identifier for this media item in the Arr application /// [JsonPropertyName("id")] public int Id { get; set; } /// /// File system path where the media is stored /// [JsonPropertyName("path")] public string Path { get; set; } = string.Empty; /// /// Tag IDs assigned to this media item in the Arr application /// [JsonPropertyName("tags")] public List Tags { get; set; } = []; /// /// Returns a string representation for debugging /// public override string ToString() => $"{Title} (ID: {Id})"; } internal record ArrTag { [JsonPropertyName("id")] public int Id { get; set; } [JsonPropertyName("label")] public string Label { get; set; } = string.Empty; } } ================================================ FILE: Tubifarry/ImportLists/ArrStack/ArrSoundtrackImport.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using System.Net; using Tubifarry.Core.Utilities; namespace Tubifarry.ImportLists.ArrStack { /// /// Import list that discovers soundtracks from Arr applications (Radarr/Sonarr) using MusicBrainz. /// public class ArrSoundtrackImport : HttpImportListBase { public override string Name => "Arr-Soundtracks"; public override ProviderMessage Message => new( "MusicBrainz enforces strict rate limiting (1 request/second). " + "Large libraries may take considerable time to process. " + "Approximately 75% of requests succeed due to MusicBrainz's rate limiting policy. " + "See: https://musicbrainz.org/doc/MusicBrainz_API/Rate_Limiting", ProviderMessageType.Warning ); public override ImportListType ListType => ImportListType.Program; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours((Definition?.Settings as ArrSoundtrackImportSettings)?.RefreshInterval ?? 12); public override int PageSize => 0; private ArrSoundtrackRequestGenerator? _generator; private ArrSoundtrackImportParser? _parser; public ArrSoundtrackImport(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) { } public override IImportListRequestGenerator GetRequestGenerator() => _generator ??= new ArrSoundtrackRequestGenerator(Settings); public override IParseImportListResponse GetParser() => _parser ??= new ArrSoundtrackImportParser(Settings, _httpClient); protected override void Test(List failures) { ValidationFailure? connectionFailure = TestArrConnection(); failures!.AddIfNotNull(connectionFailure); ValidationFailure? cacheFailure = TestCacheDirectory(); failures!.AddIfNotNull(cacheFailure); } private ValidationFailure? TestArrConnection() { try { _logger.Trace("Testing connection to Arr application at: {0}", Settings.BaseUrl); HttpRequest request = new HttpRequestBuilder(Settings.BaseUrl + Settings.APIStatusEndpoint) .AddQueryParam("apikey", Settings.ApiKey) .Build(); request.AllowAutoRedirect = true; request.RequestTimeout = TimeSpan.FromSeconds(30); HttpResponse response = _httpClient.Get(request); switch (response.StatusCode) { case HttpStatusCode.OK: return null; case HttpStatusCode.Unauthorized: return new ValidationFailure("ApiKey", "Invalid API key"); case HttpStatusCode.NotFound: return new ValidationFailure("BaseUrl", "Endpoint not found. Verify URL and API paths"); default: _logger.Warn("Arr application returned unexpected status: {0}. Response: {1}", response.StatusCode, response.Content[..200]); return new ValidationFailure("BaseUrl", $"Connection failed with status: {response.StatusCode}"); } } catch (HttpException ex) { return new ValidationFailure("BaseUrl", $"Connection failed: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Unexpected error during connection test"); return new ValidationFailure(string.Empty, $"Unexpected error: {ex.Message}"); } } private ValidationFailure? TestCacheDirectory() { try { _logger.Trace("Testing cache directory: {0}", Settings.CacheDirectory); ValidationFailure? existenceFailure = PermissionTester.TestExistance(Settings.CacheDirectory, _logger); return existenceFailure ?? PermissionTester.TestReadWritePermissions(Settings.CacheDirectory, _logger); } catch (Exception ex) { _logger.Error(ex, "Error testing cache directory"); return new ValidationFailure("CacheDirectory", $"Cache directory test failed: {ex.Message}"); } } public override IEnumerable DefaultDefinitions { get { yield return GetDefinition("Radarr", GetSettings( "http://localhost:7878", "/api/v3/system/status", "/api/v3/movie")); yield return GetDefinition("Sonarr", GetSettings( "http://localhost:8989", "/api/v3/system/status", "/api/v3/series")); } } private ImportListDefinition GetDefinition(string name, ArrSoundtrackImportSettings settings) => new() { EnableAutomaticAdd = false, Name = $"{name} Soundtracks", Implementation = GetType().Name, Settings = settings }; private static ArrSoundtrackImportSettings GetSettings(string baseUrl, string apiStatusEndpoint, string apiItemEndpoint) => new() { BaseUrl = baseUrl, APIItemEndpoint = apiItemEndpoint, APIStatusEndpoint = apiStatusEndpoint }; } } ================================================ FILE: Tubifarry/ImportLists/ArrStack/ArrSoundtrackImportParser.cs ================================================ using FuzzySharp; using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; using System.Web; using System.Xml.Linq; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; namespace Tubifarry.ImportLists.ArrStack { /// /// Handles MusicBrainz API integration for finding soundtrack albums. /// internal partial class ArrSoundtrackImportParser : IParseImportListResponse { private static readonly string[] SoundtrackTerms = ["soundtrack", "ost", "score", "original soundtrack", "film score"]; private const string MusicBrainzBaseUrl = "https://musicbrainz.org/ws/2"; // MusicBrainz requires 1 request per second private static readonly SemaphoreSlim RateLimiter = new(1, 1); private const int RateLimitDelayMs = 1100; private const int SearchResultLimit = 10; private const double SimilarityThreshold = 0.80; private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; private readonly Logger _logger; private readonly IHttpClient _httpClient; private readonly CacheService _cacheService; public ArrSoundtrackImportSettings Settings { get; set; } public ArrSoundtrackImportParser(ArrSoundtrackImportSettings settings, IHttpClient httpClient) { Settings = settings; _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); _cacheService = new CacheService { CacheType = (CacheType)settings.RequestCacheType, CacheDirectory = settings.CacheDirectory, CacheDuration = TimeSpan.FromDays(settings.CacheRetentionDays) }; } public IList ParseResponse(ImportListResponse importListResponse) => ParseResponseAsync(importListResponse).GetAwaiter().GetResult(); public async Task> ParseResponseAsync(ImportListResponse importListResponse) { List results = []; if (string.IsNullOrWhiteSpace(importListResponse.Content)) { _logger.Warn("Empty response content received from Arr application"); return results; } HashSet? requiredTagIds = await ResolveTagIdsAsync(); try { await foreach (ArrMedia? media in DeserializeMediaItemsAsync(importListResponse.Content)) { if (media == null || string.IsNullOrWhiteSpace(media.Title)) continue; if (requiredTagIds != null && !media.Tags.Any(requiredTagIds.Contains)) continue; List mediaResults = await ProcessMediaItem(media); results.AddRange(mediaResults); } } catch (JsonException ex) { _logger.Error(ex, "Failed to parse JSON response from Arr application"); throw new ImportListException(importListResponse, "Invalid JSON response from Arr application", ex); } _logger.Debug($"Soundtrack discovery completed. Found {results.Count} albums from {results.GroupBy(r => r.Artist).Count()} media items"); return results; } private async Task?> ResolveTagIdsAsync() { if (!Settings.TagFilter.Any()) return null; HashSet requestedLabels = Settings.TagFilter .Select(l => l.ToLowerInvariant()) .ToHashSet(); try { string[] parts = Settings.APIItemEndpoint.TrimEnd('/').Split('/'); parts[^1] = "tag"; string tagEndpoint = string.Join('/', parts); string url = $"{Settings.BaseUrl.TrimEnd('/')}{tagEndpoint}?apikey={Settings.ApiKey}"; HttpResponse response = await _httpClient.GetAsync(new HttpRequest(url)); List? tags = JsonSerializer.Deserialize>(response.Content, JsonOptions); if (tags == null) return []; HashSet matchedIds = tags .Where(t => requestedLabels.Contains(t.Label.ToLowerInvariant())) .Select(t => t.Id) .ToHashSet(); if (matchedIds.Count == 0) _logger.Warn("Tag filter is set but none of the specified tags were found in the Arr application: {0}", Settings.TagFilter); return matchedIds; } catch (Exception ex) { _logger.Warn(ex, "Failed to fetch tags from Arr application — proceeding without tag filtering"); return null; } } private static async IAsyncEnumerable DeserializeMediaItemsAsync(string content) { await using MemoryStream stream = new(Encoding.UTF8.GetBytes(content)); await foreach (ArrMedia? item in JsonSerializer.DeserializeAsyncEnumerable(stream, JsonOptions)) { yield return item; } } private async Task> ProcessMediaItem(ArrMedia media) { string cacheKey = GenerateMediaCacheKey(media); MediaProcessingResult cachedResults = await _cacheService.FetchAndCacheAsync(cacheKey, async () => await FetchSoundtracksForMedia(media)); if (cachedResults?.ImportListItems?.Any() == true) { _logger.Trace($"Found {cachedResults.ImportListItems.Count} soundtracks for '{media.Title}'"); return cachedResults.ImportListItems; } return []; } private async Task FetchSoundtracksForMedia(ArrMedia media) { MediaProcessingResult result = new() { Media = media }; try { List searchResults = await SearchMusicBrainzSoundtracks(media.Title); if (searchResults.Count == 0) { _logger.Debug("No soundtrack matches found for '{0}'", media.Title); return result; } result.SearchResults = searchResults; List validAlbums = []; List importItems = []; foreach (MusicBrainzSearchItem searchItem in searchResults) { if (string.IsNullOrWhiteSpace(searchItem.AlbumId)) continue; MusicBrainzAlbumItem? albumDetails = await FetchAlbumDetails(searchItem.AlbumId); if (albumDetails == null) continue; validAlbums.Add(albumDetails); if (IsGoodMatch(albumDetails, media.Title)) { ImportListItemInfo importItem = CreateImportItem(searchItem, albumDetails); importItems.Add(importItem); _logger.Trace($"Added soundtrack: '{albumDetails.Title}' by '{albumDetails.Artist}' for '{media.Title}'"); } } result.AlbumDetails = validAlbums; result.ImportListItems = importItems; } catch (HttpException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable) { _logger.Warn($"MusicBrainz rate limit exceeded for '{media.Title}'"); await Task.Delay(10 * RateLimitDelayMs); } catch (Exception ex) { _logger.Error(ex, "Failed to fetch soundtracks for media item '{0}'", media.Title); } return result; } private async Task> SearchMusicBrainzSoundtracks(string title) { await RateLimiter.WaitAsync(); try { await Task.Delay(RateLimitDelayMs); string searchUrl = BuildSearchUrl(title); _logger.Trace("Searching MusicBrainz: {0}", searchUrl); HttpRequest request = new(searchUrl); HttpResponse response = await _httpClient.GetAsync(request); return ParseSearchResponse(response.Content); } finally { RateLimiter.Release(); } } private string BuildSearchUrl(string title) { string baseUrl = $"{MusicBrainzBaseUrl}/release"; if (Settings.UseStrongMusicBrainzSearch) { string normalizedTitle = NormalizeTitle(title); IEnumerable words = normalizedTitle.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries) .Where(w => w.Length > 2) .Select(w => EscapeLuceneQuery(w)); string titleQuery = string.Join(" AND ", words.Select(w => $"release:{w}")); string query = $"({titleQuery}) AND primarytype:(Album OR Single) AND secondarytype:Soundtrack"; return $"{baseUrl}?query={HttpUtility.UrlEncode(query)}&limit={SearchResultLimit}"; } else { string query = $"{title} soundtrack"; return $"{baseUrl}?query={HttpUtility.UrlEncode(query)}&limit={SearchResultLimit}"; } } private List ParseSearchResponse(string xmlContent) { try { XDocument doc = XDocument.Parse(xmlContent); XNamespace ns = "http://musicbrainz.org/ns/mmd-2.0#"; List releases = doc.Descendants(ns + "release").ToList(); return releases.Select(release => MusicBrainzSearchItem.FromXml(release, ns)) .Where(item => item != null) .ToList(); } catch (Exception ex) { _logger.Error(ex, "Failed to parse MusicBrainz search response"); return []; } } private async Task FetchAlbumDetails(string albumId) => await _cacheService.FetchAndCacheAsync(GenerateAlbumDetailsCacheKey(albumId), async () => { await RateLimiter.WaitAsync(); try { await Task.Delay(RateLimitDelayMs); string detailsUrl = $"{MusicBrainzBaseUrl}/release-group/{albumId}"; HttpRequest request = new(detailsUrl); HttpResponse response = await _httpClient.GetAsync(request); return ParseAlbumDetails(response.Content, albumId); } finally { RateLimiter.Release(); } }); private MusicBrainzAlbumItem? ParseAlbumDetails(string xmlContent, string albumId) { try { XDocument doc = XDocument.Parse(xmlContent); XNamespace ns = "http://musicbrainz.org/ns/mmd-2.0#"; XElement? releaseGroup = doc.Descendants(ns + "release-group").FirstOrDefault(); if (releaseGroup == null) return null; return MusicBrainzAlbumItem.FromXml(releaseGroup, ns); } catch (Exception ex) { _logger.Error(ex, "Failed to parse album details for ID: {0}", albumId); return null; } } private bool IsGoodMatch(MusicBrainzAlbumItem album, string originalTitle) { if (string.IsNullOrWhiteSpace(album.Title)) return false; bool hasSoundtrackType = album.SecondaryTypes?.Any(t => string.Equals(t, "Soundtrack", StringComparison.OrdinalIgnoreCase)) ?? false; if (!hasSoundtrackType) return false; string primaryType = album.PrimaryType ?? string.Empty; bool isValidPrimaryType = string.Equals(primaryType, "Album", StringComparison.OrdinalIgnoreCase) || string.Equals(primaryType, "Single", StringComparison.OrdinalIgnoreCase); if (!isValidPrimaryType) return false; bool hasCompilationType = album.SecondaryTypes?.Any(t => string.Equals(t, "Compilation", StringComparison.OrdinalIgnoreCase)) ?? false; if (hasCompilationType) return false; int similarity = Fuzz.Ratio(album.Title, originalTitle); bool containsMovieAndSoundtrack = ContainsMovieNameAndSoundtrack(album.Title, originalTitle); return similarity > SimilarityThreshold * 100 || containsMovieAndSoundtrack; } private static bool ContainsMovieNameAndSoundtrack(string releaseTitle, string movieTitle) { string lowerReleaseTitle = releaseTitle.ToLowerInvariant(); string lowerMovieTitle = movieTitle.ToLowerInvariant(); bool containsMovieName = lowerReleaseTitle.Contains(lowerMovieTitle); bool containsSoundtrackTerm = SoundtrackTerms.Any(term => lowerReleaseTitle.Contains(term, StringComparison.OrdinalIgnoreCase)); return containsMovieName && containsSoundtrackTerm; } private static string NormalizeTitle(string title) { foreach (string term in SoundtrackTerms) title = title.Replace(term, "", StringComparison.OrdinalIgnoreCase); Dictionary numberReplacements = new(StringComparer.OrdinalIgnoreCase) { { "one", "1" }, { "two", "2" }, { "three", "3" }, { "four", "4" }, { "five", "5" }, { "six", "6" }, { "seven", "7" }, { "eight", "8" }, { "nine", "9" }, { "ten", "10" } }; foreach (KeyValuePair replacement in numberReplacements) title = Regex.Replace(title, $@"\b{replacement.Key}\b", replacement.Value, RegexOptions.IgnoreCase); title = NormalizeTitleEmptyRegex().Replace(title, "").Trim(); return NormalizeTitleSpaceRegex().Replace(title, " "); } private static string EscapeLuceneQuery(string query) { if (string.IsNullOrWhiteSpace(query)) return query; foreach (string? ch in new[] { "+", "-", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", "\"", "~", "*", "?", ":", "\\", "/" }) { query = query.Replace(ch, $"\\{ch}"); } return query; } private static ImportListItemInfo CreateImportItem(MusicBrainzSearchItem searchItem, MusicBrainzAlbumItem albumDetails) => new() { Artist = albumDetails.Artist ?? searchItem.Artist ?? "Unknown Artist", ArtistMusicBrainzId = albumDetails.ArtistId ?? searchItem.ArtistId, Album = albumDetails.Title ?? searchItem.Title ?? "Unknown Album", AlbumMusicBrainzId = albumDetails.AlbumId ?? searchItem.AlbumId, ReleaseDate = albumDetails.ReleaseDate != DateTime.MinValue ? albumDetails.ReleaseDate : searchItem.ReleaseDate }; private static string GenerateMediaCacheKey(ArrMedia media) => $"media_{media.Id}_{GenerateStringHash(media.Title)}"; private static string GenerateAlbumDetailsCacheKey(string albumId) => $"album_details_{albumId}"; private static string GenerateStringHash(string input) { if (string.IsNullOrEmpty(input)) return "empty"; HashCode hash = new(); hash.Add(input); return hash.ToHashCode().ToString("x8"); } /// /// Represents the complete processing result for a media item /// private class MediaProcessingResult { public ArrMedia? Media { get; set; } public List ImportListItems { get; set; } = []; public List SearchResults { get; set; } = []; public List AlbumDetails { get; set; } = []; } [GeneratedRegex(@"[^a-zA-Z0-9\s]")] private static partial Regex NormalizeTitleEmptyRegex(); [GeneratedRegex(@"\s+")] private static partial Regex NormalizeTitleSpaceRegex(); } } ================================================ FILE: Tubifarry/ImportLists/ArrStack/ArrSoundtrackImportSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.ImportLists.ArrStack { public class ArrSoundtrackImportSettingsValidator : AbstractValidator { public ArrSoundtrackImportSettingsValidator() { // Base URL validation RuleFor(c => c.BaseUrl) .NotEmpty() .WithMessage("Base URL is required") .ValidRootUrl() .Must(url => !url.EndsWith('/')) .WithMessage("Base URL must not end with a slash"); // API Key validation RuleFor(c => c.ApiKey) .NotEmpty() .WithMessage("API key is required") .MinimumLength(25) .WithMessage("API key must be at least 25 characters"); // When using Permanent cache, require a valid CacheDirectory RuleFor(x => x.CacheDirectory) .NotEmpty() .Must((settings, path) => (settings.RequestCacheType != (int)CacheType.Permanent) || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => (type == (int)CacheType.Permanent) || Tubifarry.AverageRuntime > TimeSpan.FromDays(4) || (DateTime.UtcNow - Tubifarry.LastStarted) > TimeSpan.FromDays(5)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // Cache Retention validation RuleFor(c => c.CacheRetentionDays) .GreaterThanOrEqualTo(1) .WithMessage("Cache retention must be at least 1 day"); // API Endpoints validation RuleFor(c => c.APIItemEndpoint) .NotEmpty() .WithMessage("API Item Endpoint is required") .Must(endpoint => endpoint.StartsWith('/')) .WithMessage("API Item Endpoint must start with '/'") .Must(endpoint => endpoint.Contains("/api/")) .WithMessage("API Item Endpoint must contain '/api/'"); RuleFor(c => c.APIStatusEndpoint) .NotEmpty() .WithMessage("API Status Endpoint is required") .Must(endpoint => endpoint.StartsWith('/')) .WithMessage("API Status Endpoint must start with '/'") .Must(endpoint => endpoint.Contains("/api/")) .WithMessage("API Status Endpoint must contain '/api/'"); // Refresh Interval validation RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(0.5) .WithMessage("Refresh interval must be at least 0.5 hours"); } } public class ArrSoundtrackImportSettings : IImportListSettings { private static readonly ArrSoundtrackImportSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Base URL", Type = FieldType.Url, HelpText = "The base URL of your Arr application", Placeholder = "http://localhost:7878")] public string BaseUrl { get; set; } = string.Empty; [FieldDefinition(1, Label = "API Key", Type = FieldType.Textbox, HelpText = "API key from your Arr application settings (General tab)", Placeholder = "Enter your API key")] public string ApiKey { get; set; } = string.Empty; [FieldDefinition(2, Label = "Tag Filter", Type = FieldType.Tag, HelpText = "Only import soundtracks for media items that have at least one of these tags. Leave empty to import all.", Advanced = true)] public IEnumerable TagFilter { get; set; } = []; [FieldDefinition(3, Label = "API Item Endpoint", Type = FieldType.Textbox, HelpText = "API endpoint for fetching media items", Advanced = true, Placeholder = "/api/v3/movie")] public string APIItemEndpoint { get; set; } = string.Empty; [FieldDefinition(4, Label = "API Status Endpoint", Type = FieldType.Textbox, HelpText = "API endpoint for system status", Advanced = true, Placeholder = "/api/v3/system/status")] public string APIStatusEndpoint { get; set; } = string.Empty; [FieldDefinition(5, Label = "Use Strict Search", Type = FieldType.Checkbox, HelpText = "Use strict MusicBrainz search. When disabled, may return more results but lower accuracy", Advanced = true)] public bool UseStrongMusicBrainzSearch { get; set; } = true; [FieldDefinition(6, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching")] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(7, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory for caching MusicBrainz results", Placeholder = "/config/soundtrack-cache")] public string CacheDirectory { get; set; } = string.Empty; [FieldDefinition(8, Label = "Cache Retention", Type = FieldType.Number, HelpText = "How many days to keep cached MusicBrainz results", Unit = "Days", Advanced = true, Placeholder = "7")] public int CacheRetentionDays { get; set; } = 7; [FieldDefinition(9, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "The interval to refresh the import list. Fractional values are allowed (e.g., 1.5 for 1 hour and 30 minutes).", Unit = "hours", Advanced = true, Placeholder = "12")] public double RefreshInterval { get; set; } = 12.0; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/ImportLists/ArrStack/ArrSoundtrackRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; namespace Tubifarry.ImportLists.ArrStack { /// /// Generates HTTP requests for fetching media items from Arr applications. /// internal class ArrSoundtrackRequestGenerator(ArrSoundtrackImportSettings settings) : IImportListRequestGenerator { private readonly ArrSoundtrackImportSettings _settings = settings; public ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain chain = new(); chain.AddTier(GetPagedRequests()); return chain; } private IEnumerable GetPagedRequests() { string url = _settings.BaseUrl.TrimEnd('/') + _settings.APIItemEndpoint; string urlWithAuth = $"{url}?apikey={_settings.ApiKey}&excludeLocalCovers=true"; yield return new ImportListRequest(urlWithAuth, HttpAccept.Json); } } } ================================================ FILE: Tubifarry/ImportLists/LastFmRecommendation/LastFmRecomendRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.LastFm; namespace Tubifarry.ImportLists.LastFmRecommendation { public class LastFmRecomendRequestGenerator(LastFmRecommendSettings settings) : IImportListRequestGenerator { private readonly LastFmRecommendSettings _settings = settings; public virtual ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain pageableRequests = new(); pageableRequests.Add(GetPagedRequests()); return pageableRequests; } private IEnumerable GetPagedRequests() { string method = _settings.Method switch { (int)LastFmRecommendMethodList.TopAlbums => "user.gettopalbums", (int)LastFmRecommendMethodList.TopArtists => "user.getTopArtists", _ => "user.getTopTracks" }; string period = _settings.Period switch { (int)LastFmUserTimePeriod.LastWeek => "7day", (int)LastFmUserTimePeriod.LastMonth => "1month", (int)LastFmUserTimePeriod.LastThreeMonths => "3month", (int)LastFmUserTimePeriod.LastSixMonths => "6month", (int)LastFmUserTimePeriod.LastTwelveMonths => "12month", _ => "overall" }; HttpRequest request = new HttpRequestBuilder(_settings.BaseUrl) .AddQueryParam("api_key", _settings.ApiKey) .AddQueryParam("method", method) .AddQueryParam("user", _settings.UserId) .AddQueryParam("period", period) .AddQueryParam("limit", _settings.FetchCount) .AddQueryParam("format", "json") .Accept(HttpAccept.Json) .Build(); yield return new ImportListRequest(request); } } } ================================================ FILE: Tubifarry/ImportLists/LastFmRecommendation/LastFmRecommend.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Parser; using System.Net; namespace Tubifarry.ImportLists.LastFmRecommendation { internal class LastFmRecommend : HttpImportListBase { private readonly IHttpClient _client; public override string Name => "Last.fm Recommend"; public override TimeSpan MinRefreshInterval => TimeSpan.FromDays(7); public override ImportListType ListType => ImportListType.LastFm; public override int PageSize => 100; public override TimeSpan RateLimit => TimeSpan.FromSeconds(5); public LastFmRecommend(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) => _client = httpClient; public override IImportListRequestGenerator GetRequestGenerator() => new LastFmRecomendRequestGenerator(Settings); public override IParseImportListResponse GetParser() => new LastFmRecommendParser(Settings, _client); protected override void Test(List failures) { failures!.AddIfNotNull(TestConnection()); } protected override ValidationFailure? TestConnection() { try { IImportListRequestGenerator generator = GetRequestGenerator(); ImportListRequest listItems = generator.GetListItems().GetAllTiers().First().First(); ImportListResponse response = FetchImportListResponse(listItems); // Validate HTTP status first if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { return new ValidationFailure(string.Empty, "Connection failed: Server returned HTTP " + $"{(int)response.HttpResponse.StatusCode} ({response.HttpResponse.StatusCode})"); } // Enhanced content type validation string? contentType = response.HttpResponse.Headers.ContentType; if (contentType == null || !IsJsonContentType(contentType)) { string receivedType = contentType ?? "null/no-content-type"; return new ValidationFailure(string.Empty, $"Unexpected content type: {receivedType}. " + "Server must return JSON (application/json or similar)"); } return null; } catch (RequestLimitReachedException) { _logger.Warn("API request limit reached"); return new ValidationFailure(string.Empty, "API rate limit exceeded - try again later"); } catch (UnsupportedFeedException ex) { _logger.Warn(ex, "Feed format not supported"); return new ValidationFailure(string.Empty, $"Unsupported feed format: {ex.Message}"); } catch (ImportListException ex) { _logger.Warn(ex, "Connection failed"); return new ValidationFailure(string.Empty, $"Connection error: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Critical connection failure"); return new ValidationFailure(string.Empty, "Configuration error - check logs for details"); } } private static bool IsJsonContentType(string mediaType) => mediaType.Equals("application/json", StringComparison.OrdinalIgnoreCase) || mediaType.Equals("text/json", StringComparison.OrdinalIgnoreCase) || mediaType.EndsWith("+json", StringComparison.OrdinalIgnoreCase); } } ================================================ FILE: Tubifarry/ImportLists/LastFmRecommendation/LastFmRecommendParser.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.ImportLists.LastFm; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; namespace Tubifarry.ImportLists.LastFmRecommendation { internal class LastFmRecommendParser : IParseImportListResponse { private readonly LastFmRecommendSettings _settings; private readonly IHttpClient _httpClient; private readonly Logger _logger; private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; public LastFmRecommendParser(LastFmRecommendSettings settings, IHttpClient httpClient) { _settings = settings; _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(ImportListResponse importListResponse) { List items = []; if (!PreProcess(importListResponse)) return items; LastFmTopResponse? jsonResponse = JsonSerializer.Deserialize(importListResponse.Content, _jsonOptions); if (jsonResponse == null) return items; if (jsonResponse.TopAlbums != null) { _logger.Trace("Processing TopAlbums response."); items.AddRange(ProcessTopAlbumsResponse(jsonResponse)); } else if (jsonResponse.TopArtists != null) { _logger.Trace("Processing TopArtists response."); items.AddRange(ProcessTopArtistsResponse(jsonResponse)); } else if (jsonResponse.TopTracks != null) { _logger.Trace("Processing TopTracks response."); items.AddRange(ProcessTopTracksResponse(jsonResponse)); } _logger.Trace("Parsing complete. Total items: {0}", items.Count); return items; } private List ProcessTopAlbumsResponse(LastFmTopResponse jsonResponse) { List items = []; List inputArtists = jsonResponse.TopAlbums?.Album.ConvertAll(x => x.Artist) ?? []; _logger.Trace("Found {0} input artists from TopAlbums.", inputArtists.Count); List> similarLists = []; foreach (LastFmArtist artist in inputArtists) { HttpRequest request = BuildRequest("artist.getSimilar", new Dictionary { { "artist", artist.Name } }); ImportListResponse response = FetchImportListResponse(request); LastFmSimilarArtistsResponse? similarResponse = JsonSerializer.Deserialize(response.Content, _jsonOptions); List similarList = similarResponse?.SimilarArtists?.Artist ?? []; similarLists.Add(similarList); _logger.Trace("Artist '{0}': Fetched {1} similar artists.", artist.Name, similarList.Count); } List<(LastFmArtist artist, int count, int bestRank)> sortedRecommended = GroupAndSortSimilarArtists(similarLists); _logger.Trace("Grouped similar artists. Unique recommendations: {0}", sortedRecommended.Count); int overallCap = _settings.FetchCount * _settings.ImportCount; int totalAlbums = 0; foreach ((LastFmArtist artist, int count, int bestRank) in sortedRecommended) { int albumLimitForArtist = Math.Min(count, 5); List albums = FetchTopAlbumsForArtist(artist); foreach (LastFmAlbum? album in albums.Take(albumLimitForArtist)) { items.Add(ConvertAlbumToImportListItems(album)); totalAlbums++; if (totalAlbums >= overallCap) { _logger.Trace("Overall album cap of {0} reached.", overallCap); break; } } if (totalAlbums >= overallCap) break; } return items; } private List ProcessTopArtistsResponse(LastFmTopResponse jsonResponse) { List items = []; List? inputArtists = jsonResponse.TopArtists?.Artist ?? []; _logger.Trace("Found {0} input artists from TopArtists.", inputArtists.Count); List> similarLists = []; foreach (LastFmArtist artist in inputArtists) { HttpRequest request = BuildRequest("artist.getSimilar", new Dictionary { { "artist", artist.Name } }); ImportListResponse response = FetchImportListResponse(request); LastFmSimilarArtistsResponse? similarResponse = JsonSerializer.Deserialize(response.Content, _jsonOptions); List similarList = similarResponse?.SimilarArtists?.Artist ?? []; similarLists.Add(similarList); _logger.Trace("Artist '{0}': Fetched {1} similar artists.", artist.Name, similarList.Count); } List<(LastFmArtist artist, int count, int bestRank)> sortedRecommended = GroupAndSortSimilarArtists(similarLists); _logger.Trace("Grouped similar artists. Unique recommendations: {0}", sortedRecommended.Count); int overallCap = _settings.FetchCount * _settings.ImportCount; int totalArtists = 0; foreach ((LastFmArtist artist, int count, int bestRank) in sortedRecommended) { items.Add(new ImportListItemInfo { Artist = artist.Name, ArtistMusicBrainzId = artist.Mbid }); totalArtists++; if (totalArtists >= overallCap) break; } return items; } private List ProcessTopTracksResponse(LastFmTopResponse jsonResponse) { List items = []; List? inputTracks = jsonResponse.TopTracks?.Track ?? []; _logger.Trace("Found {0} input tracks from TopTracks.", inputTracks.Count); List> similarLists = []; foreach (LastFmTrack track in inputTracks) { HttpRequest request = BuildRequest("track.getSimilar", new Dictionary { { "artist", track.Artist.Name }, { "track", track.Name } }); ImportListResponse response = FetchImportListResponse(request); LastFmSimilarTracksResponse? similarResponse = JsonSerializer.Deserialize(response.Content, _jsonOptions); List similarList = similarResponse?.SimilarTracks?.Track.ConvertAll(t => t.Artist) ?? []; similarLists.Add(similarList); _logger.Trace("Track '{0}' by '{1}': Fetched {2} similar artists.", track.Name, track.Artist.Name, similarList.Count); } List<(LastFmArtist artist, int count, int bestRank)> sortedRecommended = GroupAndSortSimilarArtists(similarLists); _logger.Trace("Grouped similar artists. Unique recommendations: {0}", sortedRecommended.Count); int overallCap = _settings.FetchCount * _settings.ImportCount; int totalArtists = 0; foreach ((LastFmArtist artist, int count, int bestRank) in sortedRecommended) { items.Add(new ImportListItemInfo { Artist = artist.Name, ArtistMusicBrainzId = artist.Mbid }); totalArtists++; if (totalArtists >= overallCap) break; } return items; } private static List<(LastFmArtist artist, int count, int bestRank)> GroupAndSortSimilarArtists(List> similarLists) { Dictionary recommendedDict = []; int maxRank = similarLists.Count != 0 ? similarLists.Max(list => list.Count) : 0; for (int rank = 0; rank < maxRank; rank++) { foreach (List list in similarLists) { if (list.Count > rank) { LastFmArtist simArtist = list[rank]; string key = !string.IsNullOrEmpty(simArtist.Mbid) ? simArtist.Mbid : simArtist.Name; if (recommendedDict.TryGetValue(key, out (LastFmArtist artist, int count, int bestRank) entry)) { entry.count++; entry.bestRank = Math.Min(entry.bestRank, rank); recommendedDict[key] = entry; } else { recommendedDict[key] = (simArtist, 1, rank); } } } } return [.. recommendedDict.Values .OrderByDescending(x => x.count) .ThenBy(x => x.bestRank)]; } private List FetchTopAlbumsForArtist(LastFmArtist artist) { _logger.Trace("Fetching top albums for artist '{0}'.", artist.Name); HttpRequest request = BuildRequest("artist.gettopalbums", new Dictionary { { "artist", artist.Name } }); ImportListResponse response = FetchImportListResponse(request); return JsonSerializer.Deserialize(response.Content, _jsonOptions)?.TopAlbums?.Album ?? []; } private static ImportListItemInfo ConvertAlbumToImportListItems(LastFmAlbum album) => new() { Album = album.Name, AlbumMusicBrainzId = album.Mbid, Artist = album.Artist.Name, ArtistMusicBrainzId = album.Artist.Mbid }; private HttpRequest BuildRequest(string method, Dictionary parameters) { HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .AddQueryParam("api_key", _settings.ApiKey) .AddQueryParam("method", method) .AddQueryParam("limit", _settings.ImportCount) .AddQueryParam("format", "json") .WithRateLimit(5) .Accept(HttpAccept.Json); foreach (KeyValuePair param in parameters) requestBuilder.AddQueryParam(param.Key, param.Value); _logger.Trace("Built request for method '{0}' with parameters: {1}", method, string.Join(", ", parameters.Select(p => $"{p.Key}={p.Value}"))); return requestBuilder.Build(); } protected virtual ImportListResponse FetchImportListResponse(HttpRequest request) { _logger.Trace("Fetching API response from {0}", request.Url); return new ImportListResponse(new ImportListRequest(request), _httpClient.Execute(request)); } protected virtual bool PreProcess(ImportListResponse importListResponse) { if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) throw new ImportListException(importListResponse, "Unexpected StatusCode [{0}]", importListResponse.HttpResponse.StatusCode); if (importListResponse.HttpResponse.Headers.ContentType?.Contains("text/json") == true && importListResponse.HttpRequest.Headers.Accept?.Contains("text/json") == false) throw new ImportListException(importListResponse, "Server returned HTML content"); return true; } } } ================================================ FILE: Tubifarry/ImportLists/LastFmRecommendation/LastFmRecommendSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.LastFm; using NzbDrone.Core.Validation; namespace Tubifarry.ImportLists.LastFmRecommendation { public class LastFmRecommendSettingsValidator : AbstractValidator { public LastFmRecommendSettingsValidator() { // Validate that the RefreshInterval field is not empty and meets the minimum requirement RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(5) .WithMessage("Refresh interval must be at least 5 days."); // Validate that the UserId field is not empty RuleFor(c => c.UserId) .NotEmpty() .WithMessage("Last.fm UserID is required to generate recommendations"); // Validate that the fetch limit does not exceed 100 RuleFor(c => c.FetchCount) .LessThanOrEqualTo(100) .WithMessage("Cannot fetch more than 100 items"); // Validate that the import limit does not exceed 20 RuleFor(c => c.ImportCount) .LessThanOrEqualTo(20) .WithMessage("Maximum recommendation import limit is 20"); } } public class LastFmRecommendSettings : IImportListSettings { private static readonly LastFmRecommendSettingsValidator Validator = new(); public LastFmRecommendSettings() { BaseUrl = "https://ws.audioscrobbler.com/2.0/"; ApiKey = new LastFmUserSettings().ApiKey; Method = (int)LastFmRecommendMethodList.TopArtists; Period = (int)LastFmUserTimePeriod.Overall; } // Hidden API configuration public string BaseUrl { get; set; } public string ApiKey { get; set; } [FieldDefinition(0, Label = "Last.fm Username", HelpText = "Your Last.fm username to generate personalized recommendations", Placeholder = "EnterLastFMUsername")] public string UserId { get; set; } = string.Empty; [FieldDefinition(1, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "The interval to refresh the import list. Fractional values are allowed (e.g., 1.5 for 1 day and 12 hours).", Unit = "days", Advanced = true, Placeholder = "7")] public double RefreshInterval { get; set; } = 7; [FieldDefinition(2, Label = "Recommendation Source", Type = FieldType.Select, SelectOptions = typeof(LastFmRecommendMethodList), HelpText = "Type of listening data to use for recommendations (Top Artists, Albums or Tracks)")] public int Method { get; set; } [FieldDefinition(3, Label = "Time Range", Type = FieldType.Select, SelectOptions = typeof(LastFmUserTimePeriod), HelpText = "Time period to analyze for generating recommendations (Last week/3 months/6 months/All time)")] public int Period { get; set; } [FieldDefinition(4, Label = "Fetch Limit", Type = FieldType.Number, HelpText = "Number of results to pull from the top list on Last.fm")] public int FetchCount { get; set; } = 25; [FieldDefinition(5, Label = "Import Limit", Type = FieldType.Number, HelpText = "Number of recommendations per top list result to actually import to your library")] public int ImportCount { get; set; } = 3; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum LastFmRecommendMethodList { [FieldOption(Label = "Top Artists")] TopArtists = 0, [FieldOption(Label = "Top Albums")] TopAlbums = 1, [FieldOption(Label = "Top Tracks")] TopTracks = 2 } } ================================================ FILE: Tubifarry/ImportLists/LastFmRecommendation/LastFmRecords.cs ================================================ using NzbDrone.Core.ImportLists.LastFm; using System.Text.Json.Serialization; namespace Tubifarry.ImportLists.LastFmRecommendation { public record LastFmTopResponse( [property: JsonPropertyName("topartists")] LastFmArtistList? TopArtists, [property: JsonPropertyName("topalbums")] LastFmAlbumList? TopAlbums, [property: JsonPropertyName("toptracks")] LastFmTrackList? TopTracks ); public record LastFmTrackList( [property: JsonPropertyName("track")] List Track ); public record LastFmTrack( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("artist")] LastFmArtist Artist ); public record LastFmSimilarArtistsResponse( [property: JsonPropertyName("similarartists")] LastFmArtistList? SimilarArtists ); public record LastFmSimilarTracksResponse( [property: JsonPropertyName("similartracks")] LastFmTrackList? SimilarTracks ); public record LastFmTopAlbumsResponse( [property: JsonPropertyName("topalbums")] LastFmAlbumList? TopAlbums ); } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCFRecommendations/ListenBrainzCFRecommendationsImportList.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using System.Net; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCFRecommendations { public class ListenBrainzCFRecommendationsImportList : HttpImportListBase { public override string Name => "ListenBrainz Recording Recommend"; public override ImportListType ListType => ImportListType.Other; public override TimeSpan MinRefreshInterval => TimeSpan.FromDays(1); public override int PageSize => 0; public override TimeSpan RateLimit => TimeSpan.FromMilliseconds(200); public ListenBrainzCFRecommendationsImportList(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) { } public override IImportListRequestGenerator GetRequestGenerator() => new ListenBrainzCFRecommendationsRequestGenerator(Settings); public override IParseImportListResponse GetParser() => new ListenBrainzCFRecommendationsParser(); protected override bool IsValidRelease(ImportListItemInfo release) => release.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() || release.ArtistMusicBrainzId.IsNotNullOrWhiteSpace() || (!release.Album.IsNullOrWhiteSpace() || !release.Artist.IsNullOrWhiteSpace()); protected override void Test(List failures) => failures.AddIfNotNull(TestConnection()); protected override ValidationFailure TestConnection() { try { ImportListRequest? firstRequest = GetRequestGenerator() .GetListItems() .GetAllTiers() .FirstOrDefault()? .FirstOrDefault(); if (firstRequest == null) { return new ValidationFailure(string.Empty, "No requests generated, check your configuration"); } ImportListResponse response = FetchImportListResponse(firstRequest); if (response.HttpResponse.StatusCode == HttpStatusCode.NoContent) { return new ValidationFailure(string.Empty, "No recording recommendations available for this user. These are generated based on collaborative filtering and may not be available for all users"); } if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { return new ValidationFailure(string.Empty, $"Connection failed with HTTP {(int)response.HttpResponse.StatusCode} ({response.HttpResponse.StatusCode})"); } return null!; } catch (ImportListException ex) { _logger.Warn(ex, "Connection test failed"); return new ValidationFailure(string.Empty, $"Connection error: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Test connection failed"); return new ValidationFailure(string.Empty, "Configuration error, check logs for details"); } } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCFRecommendations/ListenBrainzCFRecommendationsParser.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCFRecommendations { public class ListenBrainzCFRecommendationsParser : IParseImportListResponse { private readonly Logger _logger; public ListenBrainzCFRecommendationsParser() { _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(ImportListResponse importListResponse) { if (!PreProcess(importListResponse)) return []; try { List items = ParseRecordingRecommendations(importListResponse.Content); _logger.Trace("Successfully parsed {0} recording recommendations", items.Count); return items; } catch (Exception ex) { _logger.Error(ex, "Failed to parse ListenBrainz recording recommendations"); throw new ImportListException(importListResponse, "Failed to parse response", ex); } } private List ParseRecordingRecommendations(string content) { RecordingRecommendationResponse? response = JsonSerializer.Deserialize(content, GetJsonOptions()); IReadOnlyList? recommendations = response?.Payload?.Mbids; if (recommendations?.Any() != true) { _logger.Debug("No recording recommendations available"); return []; } _logger.Trace("Processing {0} recording recommendations", recommendations.Count); return recommendations .Where(r => !string.IsNullOrWhiteSpace(r.RecordingMbid)) .Select(r => new ImportListItemInfo { AlbumMusicBrainzId = r.RecordingMbid, Album = r.RecordingMbid }) .ToList(); } private static JsonSerializerOptions GetJsonOptions() => new() { PropertyNameCaseInsensitive = true }; private bool PreProcess(ImportListResponse importListResponse) { if (importListResponse.HttpResponse.StatusCode == HttpStatusCode.NoContent) { _logger.Info("No recording recommendations available for this user"); return false; } if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { throw new ImportListException(importListResponse, "Unexpected status code {0}", importListResponse.HttpResponse.StatusCode); } return true; } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCFRecommendations/ListenBrainzCFRecommendationsRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCFRecommendations { public class ListenBrainzCFRecommendationsRequestGenerator : IImportListRequestGenerator { private readonly ListenBrainzCFRecommendationsSettings _settings; private const int MaxItemsPerRequest = 100; public ListenBrainzCFRecommendationsRequestGenerator(ListenBrainzCFRecommendationsSettings settings) => _settings = settings; public virtual ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain pageableRequests = new(); pageableRequests.Add(GetPagedRequests()); return pageableRequests; } private IEnumerable GetPagedRequests() { int requestsNeeded = (_settings.Count + MaxItemsPerRequest - 1) / MaxItemsPerRequest; return Enumerable.Range(0, requestsNeeded) .Select(page => CreateRequest(page * MaxItemsPerRequest, Math.Min(MaxItemsPerRequest, _settings.Count - (page * MaxItemsPerRequest)))) .Where(request => request != null); } private ImportListRequest CreateRequest(int offset, int count) { if (count <= 0) return null!; HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { requestBuilder.SetHeader("Authorization", $"Token {_settings.UserToken}"); } HttpRequest request = requestBuilder.Build(); request.Url = new HttpUri($"{_settings.BaseUrl}/1/cf/recommendation/user/{_settings.UserName?.Trim()}/recording?count={count}&offset={offset}"); return new ImportListRequest(request); } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCFRecommendations/ListenBrainzCFRecommendationsSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Validation; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCFRecommendations { public class ListenBrainzCFRecommendationsSettingsValidator : AbstractValidator { public ListenBrainzCFRecommendationsSettingsValidator() { RuleFor(c => c.UserName) .NotEmpty() .WithMessage("ListenBrainz username is required"); RuleFor(c => c.Count) .GreaterThan(0) .LessThanOrEqualTo(100) .WithMessage("Count must be between 1 and 100"); RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(1) .WithMessage("Refresh interval must be at least 1 day"); } } public class ListenBrainzCFRecommendationsSettings : IImportListSettings { private static readonly ListenBrainzCFRecommendationsSettingsValidator Validator = new(); public ListenBrainzCFRecommendationsSettings() { BaseUrl = "https://api.listenbrainz.org"; RefreshInterval = 7; Count = 25; } public string BaseUrl { get; set; } [FieldDefinition(0, Label = "ListenBrainz Username", HelpText = "The ListenBrainz username to fetch recording recommendations for", Placeholder = "username")] public string UserName { get; set; } = string.Empty; [FieldDefinition(1, Label = "User Token", Type = FieldType.Password, HelpText = "Optional ListenBrainz user token for authenticated requests (higher rate limits)", Advanced = true)] public string UserToken { get; set; } = string.Empty; [FieldDefinition(2, Label = "Count", Type = FieldType.Number, HelpText = "Number of recording recommendations to fetch (1-100)")] public int Count { get; set; } [FieldDefinition(3, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "Interval between refreshes in days", Unit = "days", Advanced = true)] public double RefreshInterval { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCreatedForPlaylist/ListenBrainzCreatedForPlaylistImportList.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using System.Net; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCreatedForPlaylist { public class ListenBrainzCreatedForPlaylistImportList : HttpImportListBase { public override string Name => "ListenBrainz created for you"; public override ImportListType ListType => ImportListType.Other; public override TimeSpan MinRefreshInterval => TimeSpan.FromDays(1); public override int PageSize => 0; public override TimeSpan RateLimit => TimeSpan.FromMilliseconds(200); public ListenBrainzCreatedForPlaylistImportList(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) { } public override IImportListRequestGenerator GetRequestGenerator() => new ListenBrainzCreatedForPlaylistRequestGenerator(Settings); public override IParseImportListResponse GetParser() => new ListenBrainzCreatedForPlaylistParser(Settings, new ListenBrainzCreatedForPlaylistRequestGenerator(Settings), _httpClient); protected override bool IsValidRelease(ImportListItemInfo release) => release.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() || release.ArtistMusicBrainzId.IsNotNullOrWhiteSpace() || !release.Album.IsNullOrWhiteSpace() || !release.Artist.IsNullOrWhiteSpace(); protected override void Test(List failures) => failures.AddIfNotNull(TestConnection()); protected override ValidationFailure TestConnection() { try { ImportListRequest? firstRequest = GetRequestGenerator() .GetListItems() .GetAllTiers() .FirstOrDefault()? .FirstOrDefault(); if (firstRequest == null) { return new ValidationFailure(string.Empty, "No requests generated, check your configuration"); } ImportListResponse response = FetchImportListResponse(firstRequest); if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { return new ValidationFailure(string.Empty, $"Connection failed with HTTP {(int)response.HttpResponse.StatusCode} ({response.HttpResponse.StatusCode})"); } IList items = GetParser().ParseResponse(response); return null!; } catch (ImportListException ex) { _logger.Warn(ex, "Connection test failed"); return new ValidationFailure(string.Empty, $"Connection error: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Test connection failed"); return new ValidationFailure(string.Empty, "Configuration error, check logs for details"); } } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCreatedForPlaylist/ListenBrainzCreatedForPlaylistParser.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCreatedForPlaylist { public class ListenBrainzCreatedForPlaylistParser : IParseImportListResponse { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; private readonly ListenBrainzCreatedForPlaylistSettings _settings; private readonly ListenBrainzCreatedForPlaylistRequestGenerator _requestGenerator; private readonly IHttpClient _httpClient; private readonly Logger _logger; public ListenBrainzCreatedForPlaylistParser(ListenBrainzCreatedForPlaylistSettings settings, ListenBrainzCreatedForPlaylistRequestGenerator requestGenerator, IHttpClient httpClient) { _settings = settings; _requestGenerator = requestGenerator; _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(ImportListResponse importListResponse) { if (!PreProcess(importListResponse)) return []; try { IList items = ParseCreatedForPlaylists(importListResponse.Content); _logger.Debug("Successfully parsed {0} items from ListenBrainz playlists", items.Count); return items; } catch (Exception ex) { _logger.Error(ex, "Failed to parse ListenBrainz playlists response"); throw new ImportListException(importListResponse, "Failed to parse response", ex); } } private IList ParseCreatedForPlaylists(string content) { PlaylistsResponse? response = JsonSerializer.Deserialize(content, _jsonOptions); IReadOnlyList? playlists = response?.Playlists; if (playlists?.Any() != true) { _logger.Debug("No playlists found in response"); return []; } string targetPlaylistType = _requestGenerator.GetPlaylistTypeName(); List matchingPlaylists = playlists .Where(p => IsTargetPlaylistType(p, targetPlaylistType)) .ToList(); if (matchingPlaylists.Count == 0) { _logger.Debug("No playlists of type {0} found", targetPlaylistType); return []; } List allItems = []; foreach (PlaylistInfo? playlist in matchingPlaylists) { try { string? identifier = playlist.Playlist?.Identifier; if (string.IsNullOrWhiteSpace(identifier)) continue; _logger.Debug("Processing playlist of type {0}: {1}", targetPlaylistType, identifier); IList playlistItems = FetchPlaylistItems(ExtractPlaylistMbid(identifier)); allItems.AddRange(playlistItems); } catch (Exception ex) { _logger.Warn(ex, "Failed to process playlist"); } } return allItems; } private bool IsTargetPlaylistType(PlaylistInfo playlist, string targetType) { try { Dictionary? extension = playlist.Playlist?.Extension; if (extension?.ContainsKey("https://musicbrainz.org/doc/jspf#playlist") != true) return false; JsonElement meta = extension["https://musicbrainz.org/doc/jspf#playlist"]; string? sourcePatch = meta.GetProperty("additional_metadata") .GetProperty("algorithm_metadata") .GetProperty("source_patch") .GetString(); return sourcePatch == targetType; } catch (Exception ex) { _logger.Debug(ex, "Error checking playlist type"); return false; } } private static string ExtractPlaylistMbid(string identifier) => identifier.Split('/').Last(); private List FetchPlaylistItems(string mbid) { try { HttpRequestBuilder request = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { request.SetHeader("Authorization", $"Token {_settings.UserToken}"); } HttpRequest httpRequest = request.Build(); httpRequest.Url = new HttpUri($"{_settings.BaseUrl}/1/playlist/{mbid}"); HttpResponse response = _httpClient.Execute(httpRequest); if (response.StatusCode != HttpStatusCode.OK) { _logger.Warn("Failed to fetch playlist {0} with HTTP {1}", mbid, response.StatusCode); return []; } PlaylistResponse? playlistResponse = JsonSerializer.Deserialize(response.Content, _jsonOptions); IReadOnlyList? tracks = playlistResponse?.Playlist?.Tracks; if (tracks?.Any() != true) { _logger.Debug("No tracks found in playlist {0}", mbid); return []; } _logger.Trace("Processing {0} tracks from playlist {1}", tracks.Count, mbid); return tracks .Select(ExtractAlbumInfo) .Where(item => item != null) .Cast() .GroupBy(item => new { item.Album, item.Artist, item.ArtistMusicBrainzId }) .Select(g => g.First()) .ToList(); } catch (Exception ex) { _logger.Warn(ex, "Failed to fetch playlist {0}", mbid); return []; } } private ImportListItemInfo? ExtractAlbumInfo(TrackData track) { try { string? album = track.Album; string? artist = track.Creator; if (string.IsNullOrWhiteSpace(album) || string.IsNullOrWhiteSpace(artist)) return null; string? artistMbid = ExtractArtistMbid(track); return new ImportListItemInfo { Album = album, Artist = artist, ArtistMusicBrainzId = artistMbid }; } catch (Exception ex) { _logger.Debug(ex, "Failed to extract album info from track"); return null; } } private string? ExtractArtistMbid(TrackData track) { try { Dictionary? extension = track.Extension; if (extension?.ContainsKey("https://musicbrainz.org/doc/jspf#track") != true) return null; JsonElement trackMeta = extension["https://musicbrainz.org/doc/jspf#track"]; JsonElement.ArrayEnumerator artists = trackMeta.GetProperty("additional_metadata") .GetProperty("artists") .EnumerateArray(); JsonElement firstArtist = artists.FirstOrDefault(); if (firstArtist.ValueKind == JsonValueKind.Undefined) return null; if (firstArtist.TryGetProperty("artist_mbid", out JsonElement mbidElement)) { return mbidElement.GetString(); } return null; } catch (Exception ex) { _logger.Debug(ex, "Failed to extract artist MBID from track"); return null; } } private static bool PreProcess(ImportListResponse importListResponse) { if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { throw new ImportListException(importListResponse, "Unexpected status code {0}", importListResponse.HttpResponse.StatusCode); } return true; } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCreatedForPlaylist/ListenBrainzCreatedForPlaylistRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCreatedForPlaylist { public class ListenBrainzCreatedForPlaylistRequestGenerator : IImportListRequestGenerator { private readonly ListenBrainzCreatedForPlaylistSettings _settings; private const int DefaultPlaylistsPerCall = 25; public ListenBrainzCreatedForPlaylistRequestGenerator(ListenBrainzCreatedForPlaylistSettings settings) => _settings = settings; public virtual ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain pageableRequests = new(); pageableRequests.Add(GetPagedRequests()); return pageableRequests; } private IEnumerable GetPagedRequests() => (List) [ CreateRequest(0, DefaultPlaylistsPerCall) ]; private ImportListRequest CreateRequest(int offset, int count) { if (count <= 0) return null!; HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { requestBuilder.SetHeader("Authorization", $"Token {_settings.UserToken}"); } HttpRequest request = requestBuilder.Build(); request.Url = new HttpUri($"{_settings.BaseUrl}/1/user/{_settings.UserName?.Trim()}/playlists/createdfor?count={count}&offset={offset}"); return new ImportListRequest(request); } public string GetPlaylistTypeName() => _settings.PlaylistType switch { (int)ListenBrainzPlaylistType.DailyJams => "daily-jams", (int)ListenBrainzPlaylistType.WeeklyJams => "weekly-jams", (int)ListenBrainzPlaylistType.WeeklyExploration => "weekly-exploration", (int)ListenBrainzPlaylistType.WeeklyNew => "weekly-new", (int)ListenBrainzPlaylistType.MonthlyExploration => "monthly-exploration", _ => "daily-jams" }; } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzCreatedForPlaylist/ListenBrainzCreatedForPlaylistSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Validation; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzCreatedForPlaylist { public class ListenBrainzCreatedForPlaylistSettingsValidator : AbstractValidator { public ListenBrainzCreatedForPlaylistSettingsValidator() { RuleFor(c => c.UserName) .NotEmpty() .WithMessage("ListenBrainz username is required"); RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(1) .WithMessage("Refresh interval must be at least 1 day"); } } public class ListenBrainzCreatedForPlaylistSettings : IImportListSettings { private static readonly ListenBrainzCreatedForPlaylistSettingsValidator Validator = new(); public ListenBrainzCreatedForPlaylistSettings() { BaseUrl = "https://api.listenbrainz.org"; RefreshInterval = 7; PlaylistType = (int)ListenBrainzPlaylistType.DailyJams; } public string BaseUrl { get; set; } [FieldDefinition(0, Label = "ListenBrainz Username", HelpText = "The ListenBrainz username to fetch playlists from", Placeholder = "username")] public string UserName { get; set; } = string.Empty; [FieldDefinition(1, Label = "User Token", Type = FieldType.Password, HelpText = "Optional ListenBrainz user token for authenticated requests (higher rate limits)", Advanced = true)] public string UserToken { get; set; } = string.Empty; [FieldDefinition(2, Label = "Playlist Type", Type = FieldType.Select, SelectOptions = typeof(ListenBrainzPlaylistType), HelpText = "Type of created-for playlist to fetch")] public int PlaylistType { get; set; } [FieldDefinition(4, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "Interval between refreshes in days", Unit = "days", Advanced = true)] public double RefreshInterval { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum ListenBrainzPlaylistType { [FieldOption(Label = "Daily Jams")] DailyJams = 0, [FieldOption(Label = "Weekly Jams")] WeeklyJams = 1, [FieldOption(Label = "Weekly Exploration")] WeeklyExploration = 2, [FieldOption(Label = "Weekly New")] WeeklyNew = 3, [FieldOption(Label = "Monthly Exploration")] MonthlyExploration = 4 } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzPlaylist/ListenBrainzPlaylistImportList.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; using Tubifarry.Core.Model; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzPlaylist { public class ListenBrainzPlaylistImportList : HttpImportListBase, IPlaylistTrackSource { private static object _currentOperation = new(); public override string Name => "ListenBrainz Playlists"; public override ImportListType ListType => ImportListType.Other; public override TimeSpan MinRefreshInterval => TimeSpan.FromDays(1); public override int PageSize => 0; public override TimeSpan RateLimit => TimeSpan.FromMilliseconds(200); public ListenBrainzPlaylistImportList(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) { } public override IImportListRequestGenerator GetRequestGenerator() => new ListenBrainzPlaylistRequestGenerator(Settings); public override IParseImportListResponse GetParser() => new ListenBrainzPlaylistParser(Settings); protected override bool IsValidRelease(ImportListItemInfo release) => release.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() || release.ArtistMusicBrainzId.IsNotNullOrWhiteSpace() || !release.Album.IsNullOrWhiteSpace() || !release.Artist.IsNullOrWhiteSpace(); protected override ValidationFailure TestConnection() { try { ListenBrainzPlaylistRequestGenerator generator = new(Settings); ImportListRequest discoveryRequest = generator.CreateDiscoveryRequest(1, 0); ImportListResponse response = FetchImportListResponse(discoveryRequest); if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { return new ValidationFailure(string.Empty, $"Connection failed with HTTP {(int)response.HttpResponse.StatusCode} ({response.HttpResponse.StatusCode})"); } return null!; } catch (ImportListException ex) { _logger.Warn(ex, "Connection test failed"); return new ValidationFailure(string.Empty, $"Connection error: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Test connection failed"); return new ValidationFailure(string.Empty, "Configuration error, check logs for details"); } } public override object RequestAction(string action, IDictionary query) { if (action == "getPlaylists") { if (Settings.AccessToken.IsNullOrWhiteSpace()) { return null!; } try { List playlists = FetchAvailablePlaylists(); return new { options = new { user = Settings.AccessToken, playlists = playlists.OrderBy(p => p.name) } }; } catch (Exception ex) { _logger.Warn(ex, "Error fetching playlists from ListenBrainz"); return null!; } } return null!; } private List FetchAvailablePlaylists() { List allPlaylists = []; int offset = 0; const int count = 100; object thisOperationToken = new(); _currentOperation = thisOperationToken; Task.Delay(2000).GetAwaiter().GetResult(); if (thisOperationToken != _currentOperation) return null!; while (thisOperationToken == _currentOperation) { ListenBrainzPlaylistRequestGenerator generator = new(Settings); ImportListRequest request = generator.CreateDiscoveryRequest(count, offset); ImportListResponse response = FetchImportListResponse(request); if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { _logger.Warn("Failed to fetch playlists with HTTP {0}", response.HttpResponse.StatusCode); break; } PlaylistsResponse? playlistsResponse = JsonSerializer.Deserialize(response.Content, GetJsonOptions()); IReadOnlyList? playlists = playlistsResponse?.Playlists; if (playlists?.Any() != true) { break; } foreach (PlaylistInfo playlist in playlists) { PlaylistData? playlistData = playlist.Playlist; if (playlistData?.Identifier != null && playlistData.Title != null) { string id = ExtractPlaylistMbid(playlistData.Identifier); string name = playlistData.Title; allPlaylists.Add(new { id, name }); } } if (playlists.Count < count) { break; } offset += count; } return allPlaylists; } private static string ExtractPlaylistMbid(string identifier) => identifier.Split('/').Last(); private static JsonSerializerOptions GetJsonOptions() => new() { PropertyNameCaseInsensitive = true }; public List FetchTrackLevelItems() { List result = []; ListenBrainzPlaylistParser parser = new(Settings); ListenBrainzPlaylistRequestGenerator gen = new(Settings); foreach (string playlistId in Settings.PlaylistIds ?? []) { if (string.IsNullOrWhiteSpace(playlistId)) continue; try { ImportListRequest request = gen.CreatePlaylistRequest(playlistId); ImportListResponse resp = FetchImportListResponse(request); if (resp.HttpResponse.StatusCode == HttpStatusCode.OK) result.AddRange(parser.ParseTrackLevelItems(resp.Content)); else _logger.Warn("HTTP {0} fetching playlist {1}", resp.HttpResponse.StatusCode, playlistId); } catch (Exception ex) { _logger.Warn(ex, "Error fetching track-level items for playlist {0}", playlistId); } } return result; } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzPlaylist/ListenBrainzPlaylistParser.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; using Tubifarry.Core.Model; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzPlaylist { public class ListenBrainzPlaylistParser : IParseImportListResponse { private readonly Logger _logger; public ListenBrainzPlaylistParser(ListenBrainzPlaylistSettings settings) { _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(ImportListResponse importListResponse) { if (!PreProcess(importListResponse)) return []; try { List items = ParsePlaylistTracks(importListResponse.Content); _logger.Debug("Successfully parsed {0} items from ListenBrainz playlist", items.Count); return items; } catch (Exception ex) { _logger.Error(ex, "Failed to parse ListenBrainz playlist response"); throw new ImportListException(importListResponse, "Failed to parse response", ex); } } private List ParsePlaylistTracks(string content) { PlaylistResponse? playlistResponse = JsonSerializer.Deserialize(content, GetJsonOptions()); IReadOnlyList? tracks = playlistResponse?.Playlist?.Tracks; if (tracks?.Any() != true) { _logger.Debug("No tracks found in playlist response"); return []; } _logger.Trace("Processing {0} tracks from playlist", tracks.Count); return tracks .Select(ExtractAlbumInfo) .Where(item => item != null) .Cast() .GroupBy(item => new { item.Album, item.Artist, item.ArtistMusicBrainzId }) .Select(g => g.First()) .ToList(); } private ImportListItemInfo? ExtractAlbumInfo(TrackData track) { try { string? album = track.Album; string? artist = track.Creator; if (string.IsNullOrWhiteSpace(album) || string.IsNullOrWhiteSpace(artist)) return null; string? artistMbid = ExtractArtistMbid(track); return new ImportListItemInfo { Album = album, Artist = artist, ArtistMusicBrainzId = artistMbid }; } catch (Exception ex) { _logger.Debug(ex, "Failed to extract album info from track"); return null; } } public List ParseTrackLevelItems(string content) { try { PlaylistResponse? resp = JsonSerializer.Deserialize(content, GetJsonOptions()); IReadOnlyList? tracks = resp?.Playlist?.Tracks; if (tracks?.Any() != true) return []; return [.. tracks .Select(ToPlaylistItem) .Where(i => i != null) .Cast()]; } catch (Exception ex) { _logger.Error(ex, "Failed to parse track-level items from ListenBrainz playlist"); return []; } } private PlaylistItem? ToPlaylistItem(TrackData track) { if (string.IsNullOrWhiteSpace(track.Title) && string.IsNullOrWhiteSpace(track.Creator)) return null; string? recordingMbid = track.Identifier? .FirstOrDefault(id => id.Contains("musicbrainz.org/recording/")) ?.Split('/') .LastOrDefault(); return new PlaylistItem( ArtistMusicBrainzId: ExtractArtistMbid(track) ?? "", AlbumMusicBrainzId: null, ArtistName: track.Creator ?? "", AlbumTitle: track.Album, TrackTitle: track.Title, ForeignRecordingId: recordingMbid); } private string? ExtractArtistMbid(TrackData track) { try { Dictionary? extension = track.Extension; if (extension?.ContainsKey("https://musicbrainz.org/doc/jspf#track") != true) return null; JsonElement trackMeta = extension["https://musicbrainz.org/doc/jspf#track"]; JsonElement.ArrayEnumerator artists = trackMeta.GetProperty("additional_metadata") .GetProperty("artists") .EnumerateArray(); JsonElement firstArtist = artists.FirstOrDefault(); if (firstArtist.ValueKind == JsonValueKind.Undefined) return null; if (firstArtist.TryGetProperty("artist_mbid", out JsonElement mbidElement)) { return mbidElement.GetString(); } return null; } catch (Exception ex) { _logger.Error(ex, "Failed to extract artist MBID from track"); return null; } } private static JsonSerializerOptions GetJsonOptions() => new() { PropertyNameCaseInsensitive = true }; private static bool PreProcess(ImportListResponse importListResponse) { if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { throw new ImportListException(importListResponse, "Unexpected status code {0}", importListResponse.HttpResponse.StatusCode); } return true; } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzPlaylist/ListenBrainzPlaylistRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzPlaylist { public class ListenBrainzPlaylistRequestGenerator(ListenBrainzPlaylistSettings settings) : IImportListRequestGenerator { private readonly ListenBrainzPlaylistSettings _settings = settings; public virtual ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain pageableRequests = new(); List requests = []; foreach (string playlistId in _settings.PlaylistIds) { if (!string.IsNullOrWhiteSpace(playlistId)) { requests.Add(CreatePlaylistRequest(playlistId)); } } if (requests.Count != 0) pageableRequests.Add(requests); return pageableRequests; } public ImportListRequest CreatePlaylistRequest(string playlistId) { HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { requestBuilder.SetHeader("Authorization", $"Token {_settings.UserToken}"); } HttpRequest request = requestBuilder.Build(); request.Url = new HttpUri($"{_settings.BaseUrl}/1/playlist/{playlistId}"); return new ImportListRequest(request); } public string GetEndpointUrl() { string username = _settings.AccessToken?.Trim() ?? ""; return (ListenBrainzPlaylistEndpointType)_settings.PlaylistType switch { ListenBrainzPlaylistEndpointType.Normal => $"{_settings.BaseUrl}/1/user/{username}/playlists", ListenBrainzPlaylistEndpointType.CreatedFor => $"{_settings.BaseUrl}/1/user/{username}/playlists/createdfor", ListenBrainzPlaylistEndpointType.Recommendations => $"{_settings.BaseUrl}/1/user/{username}/playlists/recommendations", _ => $"{_settings.BaseUrl}/1/user/{username}/playlists" }; } public ImportListRequest CreateDiscoveryRequest(int count = 100, int offset = 0) { HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { requestBuilder.SetHeader("Authorization", $"Token {_settings.UserToken}"); } HttpRequest request = requestBuilder.Build(); string endpointUrl = GetEndpointUrl(); request.Url = new HttpUri($"{endpointUrl}?count={count}&offset={offset}"); return new ImportListRequest(request); } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzPlaylist/ListenBrainzPlaylistSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Validation; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzPlaylist { public class ListenBrainzPlaylistSettingsValidator : AbstractValidator { public ListenBrainzPlaylistSettingsValidator() { RuleFor(c => c.AccessToken) .NotEmpty() .WithMessage("ListenBrainz username is required"); } } public class ListenBrainzPlaylistSettings : IImportListSettings { private static readonly ListenBrainzPlaylistSettingsValidator Validator = new(); public ListenBrainzPlaylistSettings() { BaseUrl = "https://api.listenbrainz.org"; PlaylistIds = []; } public string BaseUrl { get; set; } [FieldDefinition(0, Label = "Username", HelpText = "The ListenBrainz username to fetch playlists from", Placeholder = "username")] public string AccessToken { get; set; } = string.Empty; [FieldDefinition(1, Label = "User Token", Type = FieldType.Password, HelpText = "Optional ListenBrainz user token for authenticated requests (higher rate limits)", Advanced = true)] public string UserToken { get; set; } = string.Empty; [FieldDefinition(2, Label = "Playlist Type", Type = FieldType.Select, SelectOptions = typeof(ListenBrainzPlaylistEndpointType), HelpText = "Type of playlists to fetch")] public int PlaylistType { get; set; } [FieldDefinition(3, Label = "Playlists", Type = FieldType.Playlist, HelpText = "Select specific playlists to import")] public IEnumerable PlaylistIds { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum ListenBrainzPlaylistEndpointType { [FieldOption(Label = "User Playlists")] Normal = 0, [FieldOption(Label = "Created-For")] CreatedFor = 1, [FieldOption(Label = "Recommendations")] Recommendations = 2 } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzRecords.cs ================================================ using System.Text.Json; using System.Text.Json.Serialization; namespace Tubifarry.ImportLists.ListenBrainz { // User Statistics Models public record ArtistStatsResponse( [property: JsonPropertyName("payload")] ArtistStatsPayload? Payload); public record ArtistStatsPayload( [property: JsonPropertyName("artists")] IReadOnlyList? Artists, [property: JsonPropertyName("count")] int Count, [property: JsonPropertyName("total_artist_count")] int TotalArtistCount, [property: JsonPropertyName("user_id")] string? UserId); public record ArtistStat( [property: JsonPropertyName("artist_mbid")] string? ArtistMbid, [property: JsonPropertyName("artist_name")] string? ArtistName, [property: JsonPropertyName("listen_count")] int ListenCount); public record ReleaseStatsResponse( [property: JsonPropertyName("payload")] ReleaseStatsPayload? Payload); public record ReleaseStatsPayload( [property: JsonPropertyName("releases")] IReadOnlyList? Releases, [property: JsonPropertyName("count")] int Count, [property: JsonPropertyName("total_release_count")] int TotalReleaseCount, [property: JsonPropertyName("user_id")] string? UserId); public record ReleaseStat( [property: JsonPropertyName("artist_mbids")] IReadOnlyList? ArtistMbids, [property: JsonPropertyName("artist_name")] string? ArtistName, [property: JsonPropertyName("release_mbid")] string? ReleaseMbid, [property: JsonPropertyName("release_name")] string? ReleaseName, [property: JsonPropertyName("listen_count")] int ListenCount); public record ReleaseGroupStatsResponse( [property: JsonPropertyName("payload")] ReleaseGroupStatsPayload? Payload); public record ReleaseGroupStatsPayload( [property: JsonPropertyName("release_groups")] IReadOnlyList? ReleaseGroups, [property: JsonPropertyName("count")] int Count, [property: JsonPropertyName("total_release_group_count")] int TotalReleaseGroupCount, [property: JsonPropertyName("user_id")] string? UserId); public record ReleaseGroupStat( [property: JsonPropertyName("artist_mbids")] IReadOnlyList? ArtistMbids, [property: JsonPropertyName("artist_name")] string? ArtistName, [property: JsonPropertyName("release_group_mbid")] string? ReleaseGroupMbid, [property: JsonPropertyName("release_group_name")] string? ReleaseGroupName, [property: JsonPropertyName("listen_count")] int ListenCount); // Collaborative Filtering Recommendations Models public record RecordingRecommendationResponse( [property: JsonPropertyName("payload")] RecordingRecommendationPayload? Payload); public record RecordingRecommendationPayload( [property: JsonPropertyName("mbids")] IReadOnlyList? Mbids, [property: JsonPropertyName("user_name")] string? UserName, [property: JsonPropertyName("count")] int Count, [property: JsonPropertyName("total_mbid_count")] int TotalMbidCount); public record RecordingRecommendation( [property: JsonPropertyName("recording_mbid")] string? RecordingMbid, [property: JsonPropertyName("score")] double Score); // Playlist Models public record PlaylistsResponse( [property: JsonPropertyName("playlists")] IReadOnlyList? Playlists); public record PlaylistInfo( [property: JsonPropertyName("playlist")] PlaylistData? Playlist); public record PlaylistData( [property: JsonPropertyName("identifier")] string? Identifier, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("extension")] Dictionary? Extension); public record PlaylistResponse( [property: JsonPropertyName("playlist")] PlaylistResponseData? Playlist); public record PlaylistResponseData( [property: JsonPropertyName("track")] IReadOnlyList? Tracks); public record TrackData( [property: JsonPropertyName("album")] string? Album, [property: JsonPropertyName("creator")] string? Creator, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("identifier")] IReadOnlyList? Identifier, [property: JsonPropertyName("extension")] Dictionary? Extension); } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzUserStats/ListenBrainzUserStatsImportList.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using System.Net; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzUserStats { public class ListenBrainzUserStatsImportList : HttpImportListBase { public override string Name => "ListenBrainz User Stats"; public override ImportListType ListType => ImportListType.Other; public override TimeSpan MinRefreshInterval => TimeSpan.FromDays(1); public override int PageSize => 0; public override TimeSpan RateLimit => TimeSpan.FromMilliseconds(200); public ListenBrainzUserStatsImportList(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, importListStatusService, configService, parsingService, logger) { } public override IImportListRequestGenerator GetRequestGenerator() => new ListenBrainzUserStatsRequestGenerator(Settings); public override IParseImportListResponse GetParser() => new ListenBrainzUserStatsParser(Settings); protected override bool IsValidRelease(ImportListItemInfo release) => release.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() || release.ArtistMusicBrainzId.IsNotNullOrWhiteSpace() || (!release.Album.IsNullOrWhiteSpace() || !release.Artist.IsNullOrWhiteSpace()); protected override void Test(List failures) => failures.AddIfNotNull(TestConnection()); protected override ValidationFailure TestConnection() { try { ImportListRequest? firstRequest = GetRequestGenerator() .GetListItems() .GetAllTiers() .FirstOrDefault()? .FirstOrDefault(); if (firstRequest == null) { return new ValidationFailure(string.Empty, "No requests generated, check your configuration"); } ImportListResponse response = FetchImportListResponse(firstRequest); if (response.HttpResponse.StatusCode == HttpStatusCode.NoContent) { return new ValidationFailure(string.Empty, "No statistics available for this user and time range. Statistics are calculated periodically by ListenBrainz"); } if (response.HttpResponse.StatusCode != HttpStatusCode.OK) { return new ValidationFailure(string.Empty, $"Connection failed with HTTP {(int)response.HttpResponse.StatusCode} ({response.HttpResponse.StatusCode})"); } IList items = GetParser().ParseResponse(response); return null!; } catch (ImportListException ex) { _logger.Warn(ex, "Connection test failed"); return new ValidationFailure(string.Empty, $"Connection error: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Test connection failed"); return new ValidationFailure(string.Empty, "Configuration error, check logs for details"); } } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzUserStats/ListenBrainzUserStatsParser.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; using System.Net; using System.Text.Json; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzUserStats { public class ListenBrainzUserStatsParser : IParseImportListResponse { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; private readonly ListenBrainzUserStatsSettings _settings; private readonly Logger _logger; public ListenBrainzUserStatsParser(ListenBrainzUserStatsSettings settings) { _settings = settings; _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(ImportListResponse importListResponse) { if (!PreProcess(importListResponse)) return []; try { List items = _settings.StatType switch { (int)ListenBrainzStatType.Artists => ParseArtistStats(importListResponse.Content), (int)ListenBrainzStatType.Releases => ParseReleaseStats(importListResponse.Content), (int)ListenBrainzStatType.ReleaseGroups => ParseReleaseGroupStats(importListResponse.Content), _ => [] }; _logger.Debug("Successfully parsed {0} items from ListenBrainz user stats", items.Count); return items; } catch (Exception ex) { _logger.Error(ex, "Failed to parse ListenBrainz user stats response"); throw new ImportListException(importListResponse, "Failed to parse response", ex); } } private List ParseArtistStats(string content) { ArtistStatsResponse? response = JsonSerializer.Deserialize(content, _jsonOptions); IReadOnlyList? artists = response?.Payload?.Artists; if (artists?.Any() != true) { _logger.Debug("No artist stats found"); return []; } return artists .Where(artist => !string.IsNullOrWhiteSpace(artist.ArtistName)) .Select(artist => new ImportListItemInfo { Artist = artist.ArtistName, ArtistMusicBrainzId = artist.ArtistMbid }) .Where(item => !string.IsNullOrWhiteSpace(item.Artist)) .ToList(); } private List ParseReleaseStats(string content) { ReleaseStatsResponse? response = JsonSerializer.Deserialize(content, _jsonOptions); IReadOnlyList? releases = response?.Payload?.Releases; if (releases?.Any() != true) { _logger.Debug("No release stats found"); return []; } return releases .Where(release => !string.IsNullOrWhiteSpace(release.ReleaseName) && !string.IsNullOrWhiteSpace(release.ArtistName)) .Select(release => new ImportListItemInfo { Album = release.ReleaseName, Artist = release.ArtistName, ArtistMusicBrainzId = release.ArtistMbids?.FirstOrDefault(m => !string.IsNullOrWhiteSpace(m)), AlbumMusicBrainzId = release.ReleaseMbid }) .ToList(); } private List ParseReleaseGroupStats(string content) { ReleaseGroupStatsResponse? response = JsonSerializer.Deserialize(content, _jsonOptions); IReadOnlyList? releaseGroups = response?.Payload?.ReleaseGroups; if (releaseGroups?.Any() != true) { _logger.Debug("No release group stats found"); return []; } return releaseGroups .Where(rg => !string.IsNullOrWhiteSpace(rg.ReleaseGroupName) && !string.IsNullOrWhiteSpace(rg.ArtistName)) .Select(rg => new ImportListItemInfo { Album = rg.ReleaseGroupName, Artist = rg.ArtistName, ArtistMusicBrainzId = rg.ArtistMbids?.FirstOrDefault(m => !string.IsNullOrWhiteSpace(m)) }) .ToList(); } private bool PreProcess(ImportListResponse importListResponse) { if (importListResponse.HttpResponse.StatusCode == HttpStatusCode.NoContent) { _logger.Info("No statistics available for this user and time range"); return false; } if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { throw new ImportListException(importListResponse, "Unexpected status code {0}", importListResponse.HttpResponse.StatusCode); } return true; } } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzUserStats/ListenBrainzUserStatsRequestGenerator.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzUserStats { public class ListenBrainzUserStatsRequestGenerator(ListenBrainzUserStatsSettings settings) : IImportListRequestGenerator { private readonly ListenBrainzUserStatsSettings _settings = settings; private const int MaxItemsPerRequest = 100; public virtual ImportListPageableRequestChain GetListItems() { ImportListPageableRequestChain pageableRequests = new(); pageableRequests.Add(GetPagedRequests()); return pageableRequests; } private IEnumerable GetPagedRequests() { int requestsNeeded = (_settings.Count + MaxItemsPerRequest - 1) / MaxItemsPerRequest; return Enumerable.Range(0, requestsNeeded) .Select(page => CreateRequest(page * MaxItemsPerRequest, Math.Min(MaxItemsPerRequest, _settings.Count - (page * MaxItemsPerRequest)))) .Where(request => request != null); } private ImportListRequest CreateRequest(int offset, int count) { if (count <= 0) return null!; HttpRequestBuilder requestBuilder = new HttpRequestBuilder(_settings.BaseUrl) .Accept(HttpAccept.Json); if (!string.IsNullOrEmpty(_settings.UserToken)) { requestBuilder.SetHeader("Authorization", $"Token {_settings.UserToken}"); } string endpoint = GetEndpoint(); string range = GetTimeRange(); HttpRequest request = requestBuilder.Build(); request.Url = new HttpUri($"{_settings.BaseUrl}/1/stats/user/{_settings.UserName?.Trim()}/{endpoint}?count={count}&offset={offset}&range={range}"); return new ImportListRequest(request); } private string GetEndpoint() => _settings.StatType switch { (int)ListenBrainzStatType.Artists => "artists", (int)ListenBrainzStatType.Releases => "releases", (int)ListenBrainzStatType.ReleaseGroups => "release-groups", _ => "artists" }; private string GetTimeRange() => _settings.Range switch { (int)ListenBrainzTimeRange.ThisWeek => "this_week", (int)ListenBrainzTimeRange.ThisMonth => "this_month", (int)ListenBrainzTimeRange.ThisYear => "this_year", (int)ListenBrainzTimeRange.AllTime => "all_time", _ => "all_time" }; } } ================================================ FILE: Tubifarry/ImportLists/ListenBrainz/ListenBrainzUserStats/ListenBrainzUserStatsSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Validation; namespace Tubifarry.ImportLists.ListenBrainz.ListenBrainzUserStats { public class ListenBrainzUserStatsSettingsValidator : AbstractValidator { public ListenBrainzUserStatsSettingsValidator() { RuleFor(c => c.UserName) .NotEmpty() .WithMessage("ListenBrainz username is required"); RuleFor(c => c.Count) .GreaterThan(0) .LessThanOrEqualTo(100) .WithMessage("Count must be between 1 and 100"); RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(1) .WithMessage("Refresh interval must be at least 1 day"); } } public class ListenBrainzUserStatsSettings : IImportListSettings { private static readonly ListenBrainzUserStatsSettingsValidator Validator = new(); public ListenBrainzUserStatsSettings() { BaseUrl = "https://api.listenbrainz.org"; RefreshInterval = 7; Count = 25; Range = (int)ListenBrainzTimeRange.AllTime; StatType = (int)ListenBrainzStatType.Artists; } public string BaseUrl { get; set; } [FieldDefinition(0, Label = "ListenBrainz Username", HelpText = "The ListenBrainz username to fetch statistics from", Placeholder = "username")] public string UserName { get; set; } = string.Empty; [FieldDefinition(1, Label = "User Token", Type = FieldType.Password, HelpText = "Optional ListenBrainz user token for authenticated requests (higher rate limits)", Advanced = true)] public string UserToken { get; set; } = string.Empty; [FieldDefinition(2, Label = "Statistic Type", Type = FieldType.Select, SelectOptions = typeof(ListenBrainzStatType), HelpText = "Type of statistics to fetch")] public int StatType { get; set; } [FieldDefinition(3, Label = "Time Range", Type = FieldType.Select, SelectOptions = typeof(ListenBrainzTimeRange), HelpText = "Time period for statistics")] public int Range { get; set; } [FieldDefinition(4, Label = "Count", Type = FieldType.Number, HelpText = "Number of items to fetch (1-100)")] public int Count { get; set; } [FieldDefinition(5, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "Interval between refreshes in days", Unit = "days", Advanced = true)] public double RefreshInterval { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum ListenBrainzStatType { [FieldOption(Label = "Top Artists")] Artists = 0, [FieldOption(Label = "Top Releases")] Releases = 1, [FieldOption(Label = "Top Release Groups")] ReleaseGroups = 2 } public enum ListenBrainzTimeRange { [FieldOption(Label = "This Week")] ThisWeek = 0, [FieldOption(Label = "This Month")] ThisMonth = 1, [FieldOption(Label = "This Year")] ThisYear = 2, [FieldOption(Label = "All Time")] AllTime = 3 } } ================================================ FILE: Tubifarry/ImportLists/Spotify/SpotifyUserPlaylistImport.cs ================================================ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using SpotifyAPI.Web; using SpotifyAPI.Web.Models; using Tubifarry.Core.Model; using Tubifarry.ImportLists.ArrStack; using Tubifarry.ImportLists.Spotify; namespace NzbDrone.Core.ImportLists.Spotify { public class SpotifyUserPlaylistImport : SpotifyImportListBase, IPlaylistTrackSource { private const int BaseThrottleMilliseconds = 500; private const int MaxRetries = 5; private const int BaseRateLimitDelayMilliseconds = 1000; private const int MaxRateLimitDelayMilliseconds = 30000; private FileCache? _fileCache; private static readonly SemaphoreSlim _throttleSemaphore = new(1, 1); private static DateTime _lastRequestTime = DateTime.MinValue; public SpotifyUserPlaylistImport( ISpotifyProxy spotifyProxy, IMetadataRequestBuilder requestBuilder, IImportListStatusService importListStatusService, IImportListRepository importListRepository, IConfigService configService, IParsingService parsingService, IHttpClient httpClient, Logger logger) : base(spotifyProxy, requestBuilder, importListStatusService, importListRepository, configService, parsingService, httpClient, logger) { } public override string Name => "Spotify Saved Playlists"; public override ProviderMessage Message => new( "This import list will attempt to fetch all playlists saved by the authenticated Spotify user. " + "Please note that this process may take some time depending on the number of playlists and tracks. " + "If the access token is not configured or has expired, the import will fail. " + "Additionally, large playlists or frequent refreshes may impact performance or hit API rate limits. ", ProviderMessageType.Warning); public override TimeSpan MinRefreshInterval => TimeSpan.FromHours((Definition?.Settings as ArrSoundtrackImportSettings)?.RefreshInterval ?? 12); public override IList Fetch(SpotifyWebAPI api) { List result = []; if (!string.IsNullOrWhiteSpace(Settings.CacheDirectory)) _fileCache ??= new FileCache(Settings.CacheDirectory); if (Settings.AccessToken.IsNullOrWhiteSpace()) { _logger.Warn("Access token is not configured."); return result; } try { PrivateProfile profile = _spotifyProxy.GetPrivateProfile(this, api); if (profile == null) { _logger.Warn("Failed to fetch user profile from Spotify."); return result; } Paging playlistPage = GetUserPlaylistsWithRetry(api, profile.Id); if (playlistPage == null) { _logger.Warn("Failed to fetch playlists from Spotify."); return result; } _logger.Trace($"Fetched {playlistPage.Total} playlists for user {profile.DisplayName}"); ProcessPlaylists(api, playlistPage, result, profile); } catch (Exception ex) { _logger.Warn(ex, "Error fetching playlists or tracks from Spotify"); } return result; } private void ProcessPlaylists(SpotifyWebAPI api, Paging playlistPage, List result, PrivateProfile profile) { if (playlistPage == null || playlistPage.Items == null) return; string username = profile?.Id ?? "unknown_user"; foreach (SimplePlaylist playlist in playlistPage.Items) { _logger.Trace($"Processing playlist {playlist.Name} (ID: {playlist.Id})"); if (_fileCache == null) { ProcessPlaylistTracks(api, GetPlaylistTracksWithRetry(api, playlist.Id), result); continue; } string cacheKey = GenerateCacheKey(playlist.Id, username); if (_fileCache.IsCacheValid(cacheKey, TimeSpan.FromDays(Settings.CacheRetentionDays))) { if (Settings.SkipCachedPlaylists) { _logger.Trace($"Skipping cached playlist {playlist.Name} (ID: {playlist.Id})"); continue; } CachedPlaylistData? cachedData = _fileCache.GetAsync(cacheKey).GetAwaiter().GetResult(); if (cachedData != null) { result.AddRange(cachedData.ImportListItems); continue; } } ProcessPlaylistWithCache(api, playlist, result, cacheKey); } if (playlistPage.HasNextPage()) { Paging nextPage = GetNextPageWithRetry(api, playlistPage); if (nextPage != null) ProcessPlaylists(api, nextPage, result, profile!); } } private void ProcessPlaylistWithCache(SpotifyWebAPI api, SimplePlaylist playlist, List result, string cacheKey) { Paging playlistTracks = GetPlaylistTracksWithRetry(api, playlist.Id); if (playlistTracks == null) return; List playlistItems = []; ProcessPlaylistTracks(api, playlistTracks, playlistItems); CachedPlaylistData cachedDataToSave = new() { ImportListItems = playlistItems, Playlist = playlist }; _fileCache!.SetAsync(cacheKey, cachedDataToSave, TimeSpan.FromDays(Settings.CacheRetentionDays)).GetAwaiter().GetResult(); result.AddRange(playlistItems); } private void ProcessPlaylistTracks(SpotifyWebAPI api, Paging playlistTracks, List result) { if (playlistTracks?.Items == null) return; foreach (PlaylistTrack playlistTrack in playlistTracks.Items) result!.AddIfNotNull(ParsePlaylistTrack(playlistTrack)); if (playlistTracks.HasNextPage()) { Paging nextPage = GetNextPageWithRetry(api, playlistTracks); if (nextPage != null) ProcessPlaylistTracks(api, nextPage, result); } } public List FetchTrackLevelItems() { List result = []; if (Settings.AccessToken.IsNullOrWhiteSpace()) { _logger.Warn("Access token not configured."); return result; } try { using SpotifyWebAPI api = GetApi(); PrivateProfile profile = _spotifyProxy.GetPrivateProfile(this, api); if (profile == null) { _logger.Warn("Failed to fetch user profile."); return result; } Paging playlistPage = GetUserPlaylistsWithRetry(api, profile.Id); if (playlistPage != null) CollectTrackItems(api, playlistPage, result); } catch (Exception ex) { _logger.Warn(ex, "Error fetching track-level items from Spotify"); } return result; } private void CollectTrackItems(SpotifyWebAPI api, Paging page, List result) { if (page.Items == null) return; foreach (SimplePlaylist playlist in page.Items) { Paging tracks = GetPlaylistTracksWithRetry(api, playlist.Id); if (tracks != null) AppendTrackItems(api, tracks, result); } if (page.HasNextPage()) { Paging next = GetNextPageWithRetry(api, page); if (next != null) CollectTrackItems(api, next, result); } } private void AppendTrackItems(SpotifyWebAPI api, Paging tracks, List result) { if (tracks.Items == null) return; foreach (PlaylistTrack pt in tracks.Items) { if (pt?.Track?.Album == null) continue; SimpleAlbum album = pt.Track.Album; string artistName = album.Artists?.FirstOrDefault()?.Name ?? pt.Track.Artists?.FirstOrDefault()?.Name ?? ""; string trackTitle = pt.Track.Name ?? ""; if (string.IsNullOrWhiteSpace(artistName) || string.IsNullOrWhiteSpace(trackTitle)) continue; result.Add(new PlaylistItem( ArtistMusicBrainzId: "", AlbumMusicBrainzId: null, ArtistName: artistName, AlbumTitle: album.Name, TrackTitle: trackTitle)); } if (tracks.HasNextPage()) { Paging next = GetNextPageWithRetry(api, tracks); if (next != null) AppendTrackItems(api, next, result); } } private class CachedPlaylistData { public List ImportListItems { get; set; } = []; public SimplePlaylist? Playlist { get; set; } } private static string GenerateCacheKey(string playlistId, string username) { HashCode hash = new(); hash.Add(playlistId); hash.Add(username); return hash.ToHashCode().ToString("x8"); } private Paging GetUserPlaylistsWithRetry(SpotifyWebAPI api, string userId, int retryCount = 0) { try { Throttle(); return _spotifyProxy.GetUserPlaylists(this, api, userId); } catch (SpotifyException ex) when (ex.Message.Contains("[429] API rate limit exceeded")) { if (retryCount >= MaxRetries) { _logger.Error("Maximum retry attempts reached for fetching user playlists."); throw; } int delay = CalculateRateLimitDelay(retryCount); _logger.Warn($"Rate limit exceeded. Retrying in {delay} milliseconds."); Task.Delay(delay).GetAwaiter().GetResult(); return GetUserPlaylistsWithRetry(api, userId, retryCount + 1); } } private Paging GetPlaylistTracksWithRetry(SpotifyWebAPI api, string playlistId, int retryCount = 0) { try { Throttle(); return _spotifyProxy.GetPlaylistTracks(this, api, playlistId, "next, items(track(name, artists(id, name), album(id, name, release_date, release_date_precision, artists(id, name))))"); } catch (SpotifyException ex) when (ex.Message.Contains("[429] API rate limit exceeded")) { if (retryCount >= MaxRetries) { _logger.Error("Maximum retry attempts reached for fetching playlist tracks."); throw; } int delay = CalculateRateLimitDelay(retryCount); _logger.Trace($"Rate limit exceeded. Retrying in {delay} milliseconds."); Task.Delay(delay).GetAwaiter().GetResult(); return GetPlaylistTracksWithRetry(api, playlistId, retryCount + 1); } } private Paging GetNextPageWithRetry(SpotifyWebAPI api, Paging paging, int retryCount = 0) { try { Throttle(); return _spotifyProxy.GetNextPage(this, api, paging); } catch (SpotifyException ex) when (ex.Message.Contains("[429] API rate limit exceeded")) { if (retryCount >= MaxRetries) { _logger.Error("Maximum retry attempts reached for fetching the next page."); throw; } int delay = CalculateRateLimitDelay(retryCount); _logger.Trace($"Rate limit exceeded. Retrying in {delay} milliseconds."); Task.Delay(delay).GetAwaiter().GetResult(); return GetNextPageWithRetry(api, paging, retryCount + 1); } } private static void Throttle() { _throttleSemaphore.Wait(); try { TimeSpan timeSinceLastRequest = DateTime.Now - _lastRequestTime; if (timeSinceLastRequest.TotalMilliseconds < BaseThrottleMilliseconds) { int delayNeeded = BaseThrottleMilliseconds - (int)timeSinceLastRequest.TotalMilliseconds; Task.Delay(delayNeeded).GetAwaiter().GetResult(); } _lastRequestTime = DateTime.Now; } finally { _throttleSemaphore.Release(); } } private static int CalculateRateLimitDelay(int retryCount) { int delay = (int)(BaseRateLimitDelayMilliseconds * Math.Pow(2, retryCount)); delay = Math.Min(delay, MaxRateLimitDelayMilliseconds); delay = new Random().Next(delay / 2, delay); return delay; } private SpotifyImportListItemInfo? ParsePlaylistTrack(PlaylistTrack playlistTrack) { if (playlistTrack?.Track?.Album != null) { SimpleAlbum album = playlistTrack.Track.Album; string albumName = album.Name; string? artistName = album.Artists?.FirstOrDefault()?.Name ?? playlistTrack.Track?.Artists?.FirstOrDefault()?.Name; if (albumName.IsNotNullOrWhiteSpace() && artistName.IsNotNullOrWhiteSpace()) { return new SpotifyImportListItemInfo { Artist = artistName, Album = album.Name, AlbumSpotifyId = album.Id, ReleaseDate = ParseSpotifyDate(album.ReleaseDate, album.ReleaseDatePrecision) }; } } return null; } } } ================================================ FILE: Tubifarry/ImportLists/Spotify/SpotifyUserPlaylistImportSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ImportLists.Spotify; namespace Tubifarry.ImportLists.Spotify { public class SpotifyUserPlaylistImportSettingsValidator : SpotifySettingsBaseValidator { public SpotifyUserPlaylistImportSettingsValidator() : base() { RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(2) .WithMessage("Refresh interval must be at least 0.5 hours."); RuleFor(c => c.CacheDirectory) .Must(path => string.IsNullOrEmpty(path) || Directory.Exists(path)) .WithMessage("Cache directory must be a valid path or empty."); RuleFor(c => c.CacheRetentionDays) .GreaterThanOrEqualTo(1) .WithMessage("Retention time must be at least 1 day."); } } public class SpotifyUserPlaylistImportSettings : SpotifySettingsBase { protected override AbstractValidator Validator => new SpotifySettingsBaseValidator(); public override string Scope => "playlist-read-private"; [FieldDefinition(1, Label = "Refresh Interval", Type = FieldType.Textbox, HelpText = "The interval to refresh the import list. Fractional values are allowed (e.g., 1.5 for 1 hour and 30 minutes).", Unit = "hours", Advanced = true, Placeholder = "12")] public double RefreshInterval { get; set; } = 12.0; [FieldDefinition(2, Label = "Cache Directory", Type = FieldType.Path, HelpText = "The directory where cached data will be stored. If left empty, no cache will be used.", Placeholder = "/config/spotify-cache")] public string CacheDirectory { get; set; } = string.Empty; [FieldDefinition(3, Label = "Skip Cached Playlists", Type = FieldType.Checkbox, HelpText = "If enabled, playlists that are already cached will not be searched on MusicBrainz or re-imported as long as the cache is valid. Disabling this option will force a re-search on MusicBrainz.", Advanced = true)] public bool SkipCachedPlaylists { get; set; } = true; [FieldDefinition(4, Label = "Cache Retention Time", Type = FieldType.Number, HelpText = "The number of days to retain cached data.", Advanced = true, Placeholder = "7")] public int CacheRetentionDays { get; set; } = 7; } } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Indexers.DABMusic { public class DABMusicIndexer : HttpIndexerBase { private readonly IDABMusicRequestGenerator _requestGenerator; private readonly IDABMusicParser _parser; private readonly IDABMusicSessionManager _sessionManager; public override string Name => "DABMusic"; public override string Protocol => nameof(QobuzDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 50; public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); public override ProviderMessage Message => new("DABMusic provides high-quality music downloads from qobuz.", ProviderMessageType.Info); public DABMusicIndexer( IDABMusicRequestGenerator requestGenerator, IDABMusicParser parser, IDABMusicSessionManager sessionManager, IHttpClient httpClient, IIndexerStatusService statusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, statusService, configService, parsingService, logger) { _requestGenerator = requestGenerator; _parser = parser; _sessionManager = sessionManager; } protected override async Task Test(List failures) { try { DABMusicSession? session = _sessionManager.GetOrCreateSession(Settings.BaseUrl.Trim(), Settings.Email, Settings.Password, true); if (session == null) { failures.Add(new ValidationFailure("Email", "Failed to authenticate with DABMusic. Check your email and password.")); return; } _logger.Debug($"Successfully authenticated with DABMusic as {Settings.Email}"); } catch (Exception ex) { _logger.Error(ex, "Error connecting to DABMusic API"); failures.Add(new ValidationFailure("BaseUrl", ex.Message)); return; } } public override IIndexerRequestGenerator GetRequestGenerator() { _requestGenerator.SetSetting(Settings); return _requestGenerator; } public override IParseIndexerResponse GetParser() => _parser; } } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; namespace Tubifarry.Indexers.DABMusic { public class DABMusicIndexerSettingsValidator : AbstractValidator { public DABMusicIndexerSettingsValidator() { // Validate BaseUrl RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); // Validate Email RuleFor(x => x.Email) .NotEmpty().WithMessage("Email is required.") .EmailAddress().WithMessage("Must be a valid email address."); // Validate Password RuleFor(x => x.Password) .NotEmpty().WithMessage("Password is required."); // Validate SearchLimit RuleFor(x => x.SearchLimit) .InclusiveBetween(1, 100).WithMessage("Search limit must be between 1 and 100."); // Validate RequestTimeout RuleFor(x => x.RequestTimeout) .InclusiveBetween(10, 300).WithMessage("Request timeout must be between 10 and 300 seconds."); } } public class DABMusicIndexerSettings : IIndexerSettings { private static readonly DABMusicIndexerSettingsValidator _validator = new(); public DABMusicIndexerSettings() { BaseUrl = "https://dabmusic.xyz"; SearchLimit = 50; RequestTimeout = 60; Email = ""; Password = ""; } [FieldDefinition(0, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the DABMusic API instance", Placeholder = "https://dab.yeet.su")] public string BaseUrl { get; set; } [FieldDefinition(1, Label = "Email", Type = FieldType.Textbox, HelpText = "Your DABMusic account email")] public string Email { get; set; } [FieldDefinition(2, Label = "Password", Type = FieldType.Password, HelpText = "Your DABMusic account password", Privacy = PrivacyLevel.Password)] public string Password { get; set; } [FieldDefinition(3, Label = "Search Limit", Type = FieldType.Number, HelpText = "Maximum number of results to return per search", Hidden = HiddenType.Hidden, Advanced = true)] public int SearchLimit { get; set; } [FieldDefinition(4, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for requests to DABMusic API", Advanced = true)] public int RequestTimeout { get; set; } [FieldDefinition(5, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] public int? EarlyReleaseLimit { get; set; } public NzbDroneValidationResult Validate() => new(_validator.Validate(this)); } } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicParser.cs ================================================ using NLog; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Text; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.DABMusic { public interface IDABMusicParser : IParseIndexerResponse { } public class DABMusicParser : IDABMusicParser { private readonly Logger _logger; public DABMusicParser(Logger logger) => _logger = logger; public IList ParseResponse(IndexerResponse indexerResponse) { List releases = []; try { DABMusicSearchResponse? searchResponse = JsonSerializer.Deserialize( indexerResponse.Content, IndexerParserHelper.StandardJsonOptions); if (searchResponse == null) return releases; IndexerParserHelper.ProcessItems(searchResponse.Albums, CreateAlbumData, releases); IndexerParserHelper.ProcessItems(searchResponse.Tracks, CreateTrackData, releases); } catch (Exception ex) { _logger.Error(ex, "Error parsing DABMusic search response"); } return releases; } private static AlbumData CreateAlbumData(DABMusicAlbum album) { (AudioFormat format, int bitrate, int bitDepth) = GetQuality(album.AudioQuality); long estimatedSize = IndexerParserHelper.EstimateSize(0, 0, bitrate, album.TrackCount); return new("DABMusic", nameof(QobuzDownloadProtocol)) { AlbumId = $"https://www.qobuz.com/us-en/album/{SanitizeForUrl(album.Title)}-{SanitizeForUrl(album.Artist)}/{album.Id}", AlbumName = album.Title, ArtistName = album.Artist, InfoUrl = $"https://www.qobuz.com/us-en/album/{SanitizeForUrl(album.Title)}-{SanitizeForUrl(album.Artist)}/{album.Id}", TotalTracks = album.TrackCount > 0 ? album.TrackCount : 1, ReleaseDate = album.ReleaseDate ?? DateTime.Now.Year.ToString(), ReleaseDatePrecision = "day", CustomString = album.Cover!, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = estimatedSize }; } private static AlbumData CreateTrackData(DABMusicTrack track) { (AudioFormat format, int bitrate, int bitDepth) = GetQuality(track.AudioQuality); long estimatedSize = IndexerParserHelper.EstimateSize(0, track.Duration, bitrate); return new("DABMusic", nameof(QobuzDownloadProtocol)) { AlbumId = $"https://www.qobuz.com/us-en/track/{SanitizeForUrl(track.DisplayAlbum)}-{SanitizeForUrl(track.Artist)}/{track.Id}", AlbumName = track.DisplayAlbum, ArtistName = track.Artist, InfoUrl = $"https://www.qobuz.com/us-en/track/{SanitizeForUrl(track.DisplayAlbum)}-{SanitizeForUrl(track.Artist)}/{track.Id}", TotalTracks = 1, ReleaseDate = track.ReleaseDate ?? DateTime.Now.Year.ToString(), ReleaseDatePrecision = "day", Duration = track.Duration, CustomString = track.Cover ?? track.Images?.Large ?? track.Images?.Thumbnail!, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = estimatedSize }; } private static string SanitizeForUrl(string input) => input.ToLowerInvariant() .Replace(" ", "-") .Replace("&", "and") .Where(c => char.IsLetterOrDigit(c) || c == '-') .Aggregate(new StringBuilder(), (sb, c) => sb.Append(c)) .ToString() .Trim('-'); private static (AudioFormat Format, int Bitrate, int BitDepth) GetQuality(DABMusicAudioQuality? audioQuality) { if (audioQuality == null) return (AudioFormat.Unknown, 320, 0); int bitDepth = audioQuality.MaximumBitDepth; if (audioQuality.IsHiRes) return (AudioFormat.FLAC, 1411, bitDepth); if (bitDepth <= 16 && bitDepth > 0) return (AudioFormat.FLAC, 1000, bitDepth); return (AudioFormat.MP3, 320, 0); } } } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.DABMusic { #region Search & API Response Models /// /// Request data passed through the search pipeline /// public record DABMusicRequestData( [property: JsonPropertyName("baseUrl")] string BaseUrl, [property: JsonPropertyName("searchType")] string SearchType, [property: JsonPropertyName("limit")] int Limit); /// /// Main search response from DABMusic API /// public record DABMusicSearchResponse( [property: JsonPropertyName("tracks")] List? Tracks, [property: JsonPropertyName("albums")] List? Albums, [property: JsonPropertyName("pagination")] DABMusicPagination? Pagination); /// /// Pagination information from search responses /// public record DABMusicPagination( [property: JsonPropertyName("offset")] int Offset, [property: JsonPropertyName("limit")] int Limit, [property: JsonPropertyName("total")] int Total, [property: JsonPropertyName("hasMore")] bool HasMore, [property: JsonPropertyName("returned")] int Returned); /// /// Album details response wrapper /// public record DABMusicAlbumDetailsResponse( [property: JsonPropertyName("album")] DABMusicAlbum Album); /// /// Stream response containing download URL /// public record DABMusicStreamResponse( [property: JsonPropertyName("url")] string Url); #endregion Search & API Response Models #region Core Data Models /// /// Album model from DABMusic API /// public record DABMusicAlbum( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("artistId"), JsonConverter(typeof(StringConverter))] string ArtistId, [property: JsonPropertyName("cover")] string? Cover = null, [property: JsonPropertyName("releaseDate")] string? ReleaseDate = null, [property: JsonPropertyName("genre")] string? Genre = null, [property: JsonPropertyName("trackCount")] int TrackCount = 0, [property: JsonPropertyName("audioQuality")] DABMusicAudioQuality? AudioQuality = null, [property: JsonPropertyName("label")] string? Label = null, [property: JsonPropertyName("tracks")] List? Tracks = null) { [JsonIgnore] public string Year => ReleaseDate?.Length >= 4 ? ReleaseDate[..4] : "Unknown"; } /// /// Track model from DABMusic API /// public record DABMusicTrack( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("artistId"), JsonConverter(typeof(StringConverter))] string ArtistId, [property: JsonPropertyName("albumTitle")] string? AlbumTitle = null, [property: JsonPropertyName("albumId"), JsonConverter(typeof(StringConverter))] string? AlbumId = null, [property: JsonPropertyName("albumCover")] string? Cover = null, [property: JsonPropertyName("releaseDate")] string? ReleaseDate = null, [property: JsonPropertyName("duration")] int Duration = 0, [property: JsonPropertyName("genre")] string? Genre = null, [property: JsonPropertyName("trackNumber")] int TrackNumber = 0, [property: JsonPropertyName("audioQuality")] DABMusicAudioQuality? AudioQuality = null, [property: JsonPropertyName("version")] string? Version = null, [property: JsonPropertyName("label")] string? Label = null, [property: JsonPropertyName("streamable")] bool Streamable = false, [property: JsonPropertyName("images")] DABMusicImages? Images = null) { [JsonIgnore] public string DisplayAlbum => AlbumTitle ?? "Unknown Album"; [JsonIgnore] public string DurationFormatted => Duration > 0 ? TimeSpan.FromSeconds(Duration).ToString(@"mm\:ss") : "0:00"; } /// /// Audio quality information /// public record DABMusicAudioQuality( [property: JsonPropertyName("maximumBitDepth")] int MaximumBitDepth, [property: JsonPropertyName("maximumSamplingRate")] double MaximumSamplingRate, [property: JsonPropertyName("isHiRes")] bool IsHiRes); /// /// Image URLs for different sizes /// public record DABMusicImages( [property: JsonPropertyName("small")] string? Small = null, [property: JsonPropertyName("thumbnail")] string? Thumbnail = null, [property: JsonPropertyName("large")] string? Large = null); #endregion Core Data Models } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicRequestGenerator.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; namespace Tubifarry.Indexers.DABMusic { public interface IDABMusicRequestGenerator : IIndexerRequestGenerator { public void SetSetting(DABMusicIndexerSettings settings); } /// /// Generates DABMusic search requests /// public class DABMusicRequestGenerator(Logger logger, IDABMusicSessionManager sessionManager) : IDABMusicRequestGenerator { private readonly Logger _logger = logger; private readonly IDABMusicSessionManager _sessionManager = sessionManager; private DABMusicIndexerSettings? _settings; public IndexerPageableRequestChain GetRecentRequests() => new(); public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { string query = string.Join(' ', new[] { searchCriteria.AlbumQuery, searchCriteria.ArtistQuery }.Where(s => !string.IsNullOrWhiteSpace(s))); bool isSingle = searchCriteria.Albums?.FirstOrDefault()?.AlbumReleases?.Value?.Min(r => r.TrackCount) == 1; return Generate(query, isSingle); } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) => Generate(searchCriteria.ArtistQuery, false); public void SetSetting(DABMusicIndexerSettings settings) => _settings = settings; private IndexerPageableRequestChain Generate(string query, bool isSingle) { IndexerPageableRequestChain chain = new(); if (string.IsNullOrWhiteSpace(query)) { _logger.Warn("Empty query, skipping search request"); return chain; } string baseUrl = _settings!.BaseUrl.TrimEnd('/'); string url = $"{baseUrl}/api/search?q={Uri.EscapeDataString(query)}&type=album&limit={_settings.SearchLimit}"; _logger.Trace("Creating DABMusic search request: {Url}", url); chain.Add([CreateRequest(url, baseUrl, "album")]); if (isSingle) { string fallbackUrl = $"{baseUrl}/api/search?q={Uri.EscapeDataString(query)}&type=all&limit={_settings.SearchLimit}"; _logger.Trace("Adding fallback search request: {Url}", fallbackUrl); chain.AddTier([CreateRequest(fallbackUrl, baseUrl, "all")]); } return chain; } private IndexerRequest CreateRequest(string url, string baseUrl, string searchType) { HttpRequest req = new(url) { RequestTimeout = TimeSpan.FromSeconds(_settings!.RequestTimeout), ContentSummary = new DABMusicRequestData(baseUrl, searchType, _settings.SearchLimit).ToJson(), // FlareSolverr interceptor will handle protection challenges SuppressHttpError = false, LogHttpError = true }; req.Headers["User-Agent"] = Tubifarry.UserAgent; DABMusicSession? session = _sessionManager.GetOrCreateSession(baseUrl, _settings.Email, _settings.Password); if (session?.IsValid == true) { req.Headers["Cookie"] = session.SessionCookie; _logger.Trace($"Added session cookie to request for {session.Email}"); } else { _logger.Warn("No valid session available for request"); } return new IndexerRequest(req); } } } ================================================ FILE: Tubifarry/Indexers/DABMusic/DABMusicSessionHelper.cs ================================================ using NLog; using NzbDrone.Common.Http; using System.Net; using System.Text.Json; namespace Tubifarry.Indexers.DABMusic { public record DABMusicSession(string SessionCookie, DateTime ExpiryUtc, string Email, string Password, string BaseUrl) { public bool IsValid => !string.IsNullOrEmpty(SessionCookie) && DateTime.UtcNow < ExpiryUtc; public TimeSpan TimeUntilExpiry => ExpiryUtc - DateTime.UtcNow; } public record DABMusicLoginRequest(string Email, string Password); public record DABMusicUser(int Id, string Username, string Email); public interface IDABMusicSessionManager { DABMusicSession? GetOrCreateSession(string baseUrl, string email, string password, bool forceNew = false); void InvalidateSession(string email); bool HasValidSession(string email); } public class DABMusicSessionHelper(IHttpClient httpClient, Logger logger) : IDABMusicSessionManager { private readonly IHttpClient _httpClient = httpClient; private readonly Logger _logger = logger; private readonly Dictionary _sessions = []; private static readonly TimeSpan SessionExpiry = TimeSpan.FromDays(5); private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; public DABMusicSession? GetOrCreateSession(string baseUrl, string email, string password, bool forceNew = false) { if (!forceNew && _sessions.TryGetValue(email, out DABMusicSession? existing)) { if (existing.IsValid) { _logger.Trace($"Using existing session for {email}, expires in {existing.TimeUntilExpiry}"); return existing; } _logger.Debug($"Session expired for {email}, renewing with stored credentials"); return Login(existing.BaseUrl, existing.Email, existing.Password); } _logger.Trace($"Creating new session for {email}"); return Login(baseUrl, email, password); } public void InvalidateSession(string email) { if (_sessions.Remove(email)) _logger.Info("Invalidated session for {email}"); } public bool HasValidSession(string email) => _sessions.TryGetValue(email, out DABMusicSession? session) && session.IsValid; private DABMusicSession? Login(string baseUrl, string email, string password) { try { HttpRequest request = new HttpRequestBuilder($"{baseUrl.TrimEnd('/')}/api/auth/login") .SetHeader("Content-Type", "application/json") .SetHeader("User-Agent", Tubifarry.UserAgent) .Post() .Build(); request.SetContent(JsonSerializer.Serialize(new DABMusicLoginRequest(email, password), _jsonOptions)); HttpResponse response = _httpClient.Execute(request); if (response.StatusCode != HttpStatusCode.OK) { _logger.Error($"Login failed for {email} with status {response.StatusCode}: {response.Content}"); return null; } string? sessionCookie = ExtractSessionCookie(response); if (string.IsNullOrEmpty(sessionCookie)) { _logger.Error($"No session cookie received from login response for {email}"); return null; } DateTime expiry = DateTime.UtcNow.Add(SessionExpiry); DABMusicSession session = new(sessionCookie, expiry, email, password, baseUrl); _sessions[email] = session; _logger.Info($"Successfully logged in as {email}, session expires {expiry:yyyy-MM-dd HH:mm:ss} UTC"); return session; } catch (Exception ex) { _logger.Error(ex, "Error during login for {Email}", email); return null; } } private string? ExtractSessionCookie(HttpResponse response) { if (!response.Headers.ContainsKey("Set-Cookie")) { _logger.Warn("No 'Set-Cookie' header found in login response"); return null; } string[]? setCookieValues = response.Headers.GetValues("Set-Cookie"); if (setCookieValues == null) return null; foreach (string cookieHeader in setCookieValues) { int sessionIndex = cookieHeader.IndexOf("session=", StringComparison.OrdinalIgnoreCase); if (sessionIndex >= 0) { string sessionPart = cookieHeader[sessionIndex..]; int firstSemicolon = sessionPart.IndexOf(';'); return firstSemicolon > 0 ? sessionPart[..firstSemicolon] : sessionPart; } } _logger.Warn("No session cookie found in Set-Cookie headers"); return null; } } } ================================================ FILE: Tubifarry/Indexers/DownloadProtocols.cs ================================================ namespace NzbDrone.Core.Indexers { public class YoutubeDownloadProtocol : IDownloadProtocol { } public class SoulseekDownloadProtocol : IDownloadProtocol { } public class LucidaDownloadProtocol : IDownloadProtocol { } public class QobuzDownloadProtocol : IDownloadProtocol { } public class SubSonicDownloadProtocol : IDownloadProtocol { } public class AmazonMusicDownloadProtocol : IDownloadProtocol { } } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using System.Text.RegularExpressions; namespace Tubifarry.Indexers.Lucida { public partial class LucidaIndexer : HttpIndexerBase { private readonly ILucidaRequestGenerator _requestGenerator; private readonly ILucidaParser _parser; public override string Name => "Lucida"; public override string Protocol => nameof(LucidaDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 100; public override TimeSpan RateLimit => TimeSpan.FromSeconds(2); public override ProviderMessage Message => new( "Lucida is an interface for searching music across streaming services. " + "Configure your service priorities and countries in settings.", ProviderMessageType.Info); public LucidaIndexer( ILucidaRequestGenerator requestGenerator, ILucidaParser parser, IHttpClient httpClient, IIndexerStatusService statusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, statusService, configService, parsingService, logger) { _requestGenerator = requestGenerator; _parser = parser; } protected override async Task Test(List failures) { string? baseUrl = Settings.BaseUrl?.Trim(); if (string.IsNullOrWhiteSpace(baseUrl)) { failures.Add(new ValidationFailure("BaseUrl", "Base URL is required")); return; } try { HttpRequest req = new(baseUrl); req.Headers["User-Agent"] = Tubifarry.UserAgent; HttpResponse response = await _httpClient.ExecuteAsync(req); if (response.StatusCode != System.Net.HttpStatusCode.OK) { failures.Add(new ValidationFailure("BaseUrl", $"Cannot connect to Lucida instance: HTTP {(int)response.StatusCode}")); return; } if (!response.Content.Contains("Lucida") && !LucidaHeaderRegex().IsMatch(response.Content)) { failures.Add(new ValidationFailure("BaseUrl", "The provided URL does not appear to be a Lucida instance")); return; } } catch (Exception ex) { _logger.Error(ex, "Error connecting to Lucida instance"); failures.Add(new ValidationFailure("BaseUrl", ex.Message)); return; } Dictionary> services; try { services = await LucidaServiceHelper.GetServicesAsync(baseUrl, _httpClient, _logger); if (services.Count == 0) { failures.Add(new ValidationFailure("BaseUrl", "No services available from Lucida instance")); return; } } catch (Exception ex) { _logger.Error(ex, "Error fetching services"); failures.Add(new ValidationFailure("BaseUrl", ex.Message)); return; } List codes = Settings.CountryCode? .Split([';', ','], StringSplitOptions.RemoveEmptyEntries) .Select(c => c.Trim().ToUpperInvariant()) .ToList() ?? []; if (codes.Count == 0) { failures.Add(new ValidationFailure("CountryCode", "At least one country code is required")); } else { foreach (string? code in codes) { if (!services.Values.Where(x => x != null).SelectMany(x => x).Any(c => string.Equals(c.Code, code, StringComparison.OrdinalIgnoreCase))) failures.Add(new ValidationFailure("CountryCode", $"Country code '{code}' is not valid for any service")); } } } public override IIndexerRequestGenerator GetRequestGenerator() { _requestGenerator.SetSetting(Settings); return _requestGenerator; } public override IParseIndexerResponse GetParser() => _parser; [GeneratedRegex(".*?(Lucida|Music).*?", RegexOptions.IgnoreCase, "de-DE")] private static partial Regex LucidaHeaderRegex(); } } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; namespace Tubifarry.Indexers.Lucida { public class LucidaIndexerSettingsValidator : AbstractValidator { public LucidaIndexerSettingsValidator() { // Validate BaseUrl RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); // Validate service priorities RuleForEach(x => x.ServicePriorities) .Must(kvp => int.TryParse(kvp.Value, out int v) && v >= 0 && v <= 50) .WithMessage("Priority values must be between 0 and 50."); // Validate country codes RuleFor(x => x.CountryCode) .NotEmpty().WithMessage("Country code is required.") .Must(cc => cc .Split([';', ','], StringSplitOptions.RemoveEmptyEntries) .All(code => code.Trim().Length == 2)) .WithMessage("Country code must be a semicolon or comma-separated list of 2-letter country codes."); } } public class LucidaIndexerSettings : IIndexerSettings { private static readonly LucidaIndexerSettingsValidator _validator = new(); private static readonly Dictionary _defaultPriorities = new(StringComparer.OrdinalIgnoreCase) { ["Qobuz"] = "0", ["Tidal"] = "2", ["Deezer"] = "3", ["SoundCloud"] = "4", ["Amazon Music"] = "5", ["Yandex Music"] = "6" }; private List> _servicePriorities; public LucidaIndexerSettings() { CountryCode = "US"; RequestTimeout = 60; BaseUrl = "https://lucida.to"; _servicePriorities = [.. _defaultPriorities]; } [FieldDefinition(0, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the Lucida instance", Placeholder = "https://lucida.to")] public string BaseUrl { get; set; } [FieldDefinition(1, Label = "Country Code", Type = FieldType.Textbox, HelpText = "Two-letter country code for service regions separated by ;")] public string CountryCode { get; set; } [FieldDefinition(2, Label = "Service Priorities", Type = FieldType.KeyValueList, HelpText = "Define priority for music services.")] public IEnumerable> ServicePriorities { get { Dictionary>? services = LucidaServiceHelper.HasAvailableServices(BaseUrl) ? LucidaServiceHelper.GetAvailableServices(BaseUrl) : null; return _defaultPriorities.Keys.Where(displayName => services == null || (LucidaServiceHelper.GetServiceKey(displayName) is string serviceKey && services.ContainsKey(serviceKey))) .Select(displayName => new KeyValuePair(displayName, _servicePriorities.FirstOrDefault(p => string.Equals(p.Key, displayName, StringComparison.OrdinalIgnoreCase)).Value ?? _defaultPriorities[displayName])).OrderBy(kvp => int.Parse(kvp.Value)); } set { if (value != null) { Dictionary custom = value.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); _servicePriorities = _defaultPriorities.Keys.Select(displayName => new KeyValuePair(displayName, custom.TryGetValue(displayName, out string? v) ? v : _defaultPriorities[displayName])).ToList(); } else { _servicePriorities = [.. _defaultPriorities]; } } } [FieldDefinition(3, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for requests to Lucida", Advanced = true)] public int RequestTimeout { get; set; } [FieldDefinition(4, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] public int? EarlyReleaseLimit { get; set; } public NzbDroneValidationResult Validate() => new(_validator.Validate(this)); } } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Lucida { #region Search & Parsing Models /// /// Request data passed through the search pipeline /// public record LucidaRequestData( [property: JsonPropertyName("serviceValue")] string ServiceValue, [property: JsonPropertyName("baseUrl")] string BaseUrl, [property: JsonPropertyName("countryCode")] string CountryCode, [property: JsonPropertyName("isSingle")] bool IsSingle); /// /// Search results wrapper /// public record LucidaSearchResults( [property: JsonPropertyName("results")] LucidaResultsContainer Results); /// /// Results container from search /// public record LucidaResultsContainer( [property: JsonPropertyName("success")] bool Success, [property: JsonPropertyName("results")] LucidaResultsData Results); /// /// Actual search results data /// public record LucidaResultsData( [property: JsonPropertyName("query")] string Query, [property: JsonPropertyName("albums")] List? Albums = null, [property: JsonPropertyName("tracks")] List? Tracks = null, [property: JsonPropertyName("artists")] List? Artists = null); /// /// JavaScript data wrapper from page extraction /// public record LucidaDataWrapper( [property: JsonPropertyName("type")] string Type, [property: JsonPropertyName("data")] DataContainer Data); /// /// Data container for JavaScript extracted data /// public record DataContainer( [property: JsonPropertyName("results")] LucidaResultsContainer Results, [property: JsonPropertyName("query")] string? Query = null, [property: JsonPropertyName("country")] string? Country = null, [property: JsonPropertyName("service")] string? Service = null, [property: JsonPropertyName("uses")] Dictionary? Uses = null); #endregion Search & Parsing Models #region Core Data Models /// /// Album model from search results /// public record LucidaAlbum( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("releaseDate")] string ReleaseDate, [property: JsonPropertyName("trackCount"), JsonConverter(typeof(FloatConverter))] float TrackCount, [property: JsonPropertyName("coverArtwork")] List? CoverArtwork = null, [property: JsonPropertyName("artists")] List? Artists = null, [property: JsonPropertyName("upc")] string? Upc = null, [property: JsonPropertyName("label")] string? Label = null, [property: JsonPropertyName("genre")] List? Genre = null) { public LucidaAlbum() : this("", "", "", "", 0) { } } /// /// Track model from search results /// public record LucidaTrack( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("durationMs")] long DurationMs, [property: JsonPropertyName("releaseDate")] string ReleaseDate, [property: JsonPropertyName("description")] string? Description = null, [property: JsonPropertyName("explicit")] bool Explicit = false, [property: JsonPropertyName("isrc")] string? Isrc = null, [property: JsonPropertyName("trackNumber"), JsonConverter(typeof(FloatConverter))] float TrackNumber = 0, [property: JsonPropertyName("discNumber"), JsonConverter(typeof(FloatConverter))] float DiscNumber = 1, [property: JsonPropertyName("coverArtwork")] List? CoverArtwork = null, [property: JsonPropertyName("artists")] List? Artists = null, [property: JsonPropertyName("album")] LucidaAlbumReference? Album = null, [property: JsonPropertyName("genres")] List? Genres = null) { public LucidaTrack() : this("", "", "", 0, "") { } } /// /// Album reference within track info /// public record LucidaAlbumReference( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("coverArtwork")] List? CoverArtwork = null, [property: JsonPropertyName("artists")] List? Artists = null, [property: JsonPropertyName("upc")] string? Upc = null, [property: JsonPropertyName("releaseDate")] string? ReleaseDate = null, [property: JsonPropertyName("label")] string? Label = null, [property: JsonPropertyName("genre")] List? Genre = null) { public LucidaAlbumReference() : this("", "", "") { } } /// /// Artist information /// public record LucidaArtist( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("pictures")] List? Pictures = null) { public LucidaArtist() : this("", "", "") { } } /// /// Artwork/cover image information /// public record LucidaArtwork( [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("width"), JsonConverter(typeof(FloatConverter))] float Width, [property: JsonPropertyName("height"), JsonConverter(typeof(FloatConverter))] float Height) { public LucidaArtwork() : this("", 0, 0) { } [JsonIgnore] public int PixelCount => (int)(Width * Height); } /// /// Service country information /// public record ServiceCountry( [property: JsonPropertyName("code")] string Code, [property: JsonPropertyName("label")] string Name) { public ServiceCountry() : this("", "") { } } /// /// Country API response /// public record CountryResponse( [property: JsonPropertyName("success")] bool Success, [property: JsonPropertyName("countries")] List? Countries = null) { public CountryResponse() : this(false) { } } #endregion Core Data Models #region JavaScript Data Response Models /// /// Unified info model for both tracks and albums from JavaScript data /// public record LucidaInfo( [property: JsonPropertyName("success")] bool Success, [property: JsonPropertyName("type")] string Type = "", [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id = "", [property: JsonPropertyName("url")] string Url = "", [property: JsonPropertyName("title")] string Title = "", [property: JsonPropertyName("durationMs")] long DurationMs = 0, [property: JsonPropertyName("album")] LucidaAlbumRef? Album = null, [property: JsonPropertyName("isrc")] string? Isrc = null, [property: JsonPropertyName("copyright")] string? Copyright = null, [property: JsonPropertyName("trackNumber")] int TrackNumber = 0, [property: JsonPropertyName("discNumber")] int DiscNumber = 1, [property: JsonPropertyName("explicit")] bool Explicit = false, [property: JsonPropertyName("stats")] LucidaStats? Stats = null, [property: JsonPropertyName("upc")] string? Upc = null, [property: JsonPropertyName("trackCount")] int TrackCount = 0, [property: JsonPropertyName("discCount")] int DiscCount = 1, [property: JsonPropertyName("releaseDate")] string? ReleaseDate = null) { [JsonPropertyName("artists")] public LucidaArtistInfo[] Artists { get; init; } = []; [JsonPropertyName("producers")] public string[] Producers { get; init; } = []; [JsonPropertyName("composers")] public string[] Composers { get; init; } = []; [JsonPropertyName("lyricists")] public string[] Lyricists { get; init; } = []; [JsonPropertyName("coverArtwork")] public LucidaArtworkInfo[] CoverArtwork { get; init; } = []; [JsonPropertyName("tracks")] public LucidaTrackInfo[] Tracks { get; init; } = []; } /// /// Artist info from JavaScript data (different structure than search results) /// public record LucidaArtistInfo( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id = "", [property: JsonPropertyName("name")] string Name = "", [property: JsonPropertyName("url")] string Url = "") { [JsonPropertyName("pictures")] public string[] Pictures { get; init; } = []; }; /// /// Artwork info from JavaScript data /// public record LucidaArtworkInfo( [property: JsonPropertyName("url")] string Url = "", [property: JsonPropertyName("width")] int Width = 0, [property: JsonPropertyName("height")] int Height = 0) { [JsonIgnore] public int PixelCount => Width * Height; }; /// /// Album reference within track info /// public record LucidaAlbumRef( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id = "", [property: JsonPropertyName("url")] string Url = "", [property: JsonPropertyName("title")] string Title = "", [property: JsonPropertyName("releaseDate")] string? ReleaseDate = null) { [JsonPropertyName("coverArtwork")] public LucidaArtworkInfo[] CoverArtwork { get; init; } = []; }; /// /// Track info within album /// public record LucidaTrackInfo( [property: JsonPropertyName("url")] string Url = "", [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id = "", [property: JsonPropertyName("title")] string Title = "", [property: JsonPropertyName("durationMs")] long DurationMs = 0, [property: JsonPropertyName("isrc")] string? Isrc = null, [property: JsonPropertyName("copyright")] string? Copyright = null, [property: JsonPropertyName("trackNumber")] int TrackNumber = 0, [property: JsonPropertyName("discNumber")] int DiscNumber = 1, [property: JsonPropertyName("explicit")] bool Explicit = false, [property: JsonPropertyName("csrf")] string? Csrf = null, [property: JsonPropertyName("csrfFallback")] string? CsrfFallback = null) { [JsonPropertyName("artists")] public LucidaArtistInfo[] Artists { get; init; } = []; [JsonPropertyName("producers")] public string[] Producers { get; init; } = []; [JsonPropertyName("composers")] public string[] Composers { get; init; } = []; [JsonPropertyName("lyricists")] public string[] Lyricists { get; init; } = []; }; /// /// Statistics/metadata about the service /// public record LucidaStats( [property: JsonPropertyName("account")] string Account = "", [property: JsonPropertyName("country")] string Country = "", [property: JsonPropertyName("service")] string Service = "", [property: JsonPropertyName("cache")] bool Cache = false); #endregion JavaScript Data Response Models #region Application Models /// /// Track model for application use /// public class LucidaTrackModel { public string Id { get; set; } = string.Empty; public string Title { get; set; } = string.Empty; public string Artist { get; set; } = string.Empty; public List Artists { get; set; } = []; public string? AlbumTitle { get; set; } public string? AlbumId { get; set; } public string? AlbumUrl { get; set; } public long DurationMs { get; set; } public int TrackNumber { get; set; } public int DiscNumber { get; set; } = 1; public bool IsExplicit { get; set; } public string? Isrc { get; set; } public string? ReleaseDate { get; set; } public string? Year { get; set; } public string? Copyright { get; set; } public string? CoverUrl { get; set; } public List CoverArtworks { get; set; } = []; public List Composers { get; set; } = []; public List Producers { get; set; } = []; public List Lyricists { get; set; } = []; // Service URLs public string? OriginalServiceUrl { get; set; } public string? DetailPageUrl { get; set; } public string? ServiceName { get; set; } public string? Url { get; set; } // Authentication tokens public string? PrimaryToken { get; set; } public string? FallbackToken { get; set; } public long TokenExpiry { get; set; } [JsonIgnore] public bool HasValidTokens => !string.IsNullOrEmpty(PrimaryToken) && !string.IsNullOrEmpty(FallbackToken); public string GetBestCoverArtUrl() { if (CoverArtworks.Count == 0) return CoverUrl ?? string.Empty; return CoverArtworks .Where(a => a.Width > 0 && a.Height > 0) .OrderByDescending(a => a.PixelCount) .FirstOrDefault()?.Url ?? CoverUrl ?? string.Empty; } public string FormatDuration() { if (DurationMs <= 0) return "0:00"; TimeSpan timeSpan = TimeSpan.FromMilliseconds(DurationMs); return $"{(int)timeSpan.TotalMinutes}:{timeSpan.Seconds:D2}"; } } /// /// Album model for application use /// public class LucidaAlbumModel { public string Id { get; set; } = string.Empty; public string Title { get; set; } = string.Empty; public string Artist { get; set; } = string.Empty; public List Artists { get; set; } = []; public int TrackCount { get; set; } public int DiscCount { get; set; } = 1; public string? ReleaseDate { get; set; } public string? Year { get; set; } public string? Upc { get; set; } public string? Copyright { get; set; } public string? CoverUrl { get; set; } public List CoverArtworks { get; set; } = []; public List Tracks { get; set; } = []; // Service URLs public string? OriginalServiceUrl { get; set; } public string? DetailPageUrl { get; set; } public string? ServiceName { get; set; } // Authentication tokens public string? PrimaryToken { get; set; } public string? FallbackToken { get; set; } public long TokenExpiry { get; set; } [JsonIgnore] public bool HasValidTokens => !string.IsNullOrEmpty(PrimaryToken) && !string.IsNullOrEmpty(FallbackToken); [JsonIgnore] public bool IsValid => !string.IsNullOrEmpty(Title) && !string.IsNullOrEmpty(Artist) && Tracks.Count > 0; public string GetBestCoverArtUrl() { if (CoverArtworks.Count == 0) return CoverUrl ?? string.Empty; return CoverArtworks .Where(a => a.Width > 0 && a.Height > 0) .OrderByDescending(a => a.PixelCount) .FirstOrDefault()?.Url ?? CoverUrl ?? string.Empty; } public long GetTotalDurationMs() => Tracks.Sum(t => t.DurationMs); public string FormatTotalDuration() { long totalMs = GetTotalDurationMs(); if (totalMs <= 0) return "0:00"; TimeSpan timeSpan = TimeSpan.FromMilliseconds(totalMs); return timeSpan.TotalHours >= 1 ? $"{(int)timeSpan.TotalHours}h {timeSpan.Minutes}m" : $"{timeSpan.Minutes}m {timeSpan.Seconds}s"; } } /// /// Authentication tokens /// public record LucidaTokens(string Primary, string Fallback, long Expiry) { public bool IsValid => !string.IsNullOrEmpty(Primary) && !string.IsNullOrEmpty(Fallback); public static LucidaTokens Empty => new(string.Empty, string.Empty, 0); public static LucidaTokens Create(string primary, string fallback, long? expiry = null) => new(primary, fallback, expiry ?? DateTimeOffset.UtcNow.AddDays(30).ToUnixTimeSeconds()); } #endregion Application Models #region API Request/Response Models /// /// Request for download initiation /// public record LucidaDownloadRequestInfo( [property: JsonPropertyName("url")] string Url = "", [property: JsonPropertyName("metadata")] bool Metadata = true, [property: JsonPropertyName("compat")] bool Compat = false, [property: JsonPropertyName("private")] bool Private = true, [property: JsonPropertyName("handoff")] bool Handoff = true, [property: JsonPropertyName("downscale")] string Downscale = "original") { [JsonPropertyName("account")] public LucidaAccountInfo Account { get; init; } = new(); [JsonPropertyName("upload")] public LucidaUploadInfo Upload { get; init; } = new(); [JsonPropertyName("token")] public LucidaTokenData TokenData { get; init; } = new(); public static LucidaDownloadRequestInfo CreateWithTokens(string url, string primaryToken, string fallbackToken, long expiry) => new(Url: url) { TokenData = new(primaryToken, fallbackToken, expiry) }; } /// /// Account info for requests /// public record LucidaAccountInfo( [property: JsonPropertyName("type")] string Type = "country", [property: JsonPropertyName("id")] string Id = "auto"); /// /// Upload info for requests /// public record LucidaUploadInfo( [property: JsonPropertyName("enabled")] bool Enabled = false, [property: JsonPropertyName("service")] string Service = "pixeldrain"); /// /// Token data for authentication /// public record LucidaTokenData( [property: JsonPropertyName("primary")] string? Primary = null, [property: JsonPropertyName("secondary")] string? Secondary = null, [property: JsonPropertyName("expiry")] long Expiry = 0); /// /// Response from download initiation /// public record LucidaDownloadResponse( [property: JsonPropertyName("success")] bool Success, [property: JsonPropertyName("handoff")] string? Handoff = null, [property: JsonPropertyName("name")] string? Name = null, [property: JsonPropertyName("server")] string? Server = null, [property: JsonPropertyName("error")] string? Error = null, [property: JsonPropertyName("stats")] LucidaStatsResponse? Stats = null, [property: JsonPropertyName("fromExternal")] bool FromExternal = false, [property: JsonPropertyName("skipbo")] string? Skipbo = null, [property: JsonPropertyName("skipboExpiration")] long SkipboExpiration = 0); /// /// Stats in download response /// public record LucidaStatsResponse( [property: JsonPropertyName("service")] string? Service = null, [property: JsonPropertyName("account")] string? Account = null); /// /// Status response for polling /// public record LucidaStatusResponse( [property: JsonPropertyName("success")] bool Success, [property: JsonPropertyName("status")] string? Status = null, [property: JsonPropertyName("name")] string? Name = null, [property: JsonPropertyName("error")] string? Error = null, [property: JsonPropertyName("message")] string? Message = null); #endregion API Request/Response Models } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaRequestGenerator.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; namespace Tubifarry.Indexers.Lucida { public interface ILucidaRequestGenerator : IIndexerRequestGenerator { public void SetSetting(LucidaIndexerSettings settings); } /// /// Generates Lucida search requests with tiering and service checks /// public class LucidaRequestGenerator : ILucidaRequestGenerator { private readonly IHttpClient _httpClient; private readonly Logger _logger; private LucidaIndexerSettings? _settings; public LucidaRequestGenerator(IHttpClient httpClient, Logger logger) => (_httpClient, _logger) = (httpClient, logger); public IndexerPageableRequestChain GetRecentRequests() => new(); public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) => Generate( query: string.Join(' ', new[] { searchCriteria.AlbumQuery, searchCriteria.ArtistQuery }.Where(s => !string.IsNullOrWhiteSpace(s))), isSingle: searchCriteria.Albums?.FirstOrDefault()?.AlbumReleases?.Value?.Min(r => r.TrackCount) == 1); public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) => Generate(searchCriteria.ArtistQuery, false); public void SetSetting(LucidaIndexerSettings settings) => _settings = settings; private IndexerPageableRequestChain Generate(string query, bool isSingle) { IndexerPageableRequestChain chain = new(); if (string.IsNullOrWhiteSpace(query)) { _logger.Warn("Empty query, skipping search request"); return chain; } string baseUrl = _settings!.BaseUrl.TrimEnd('/'); Dictionary> services = LucidaServiceHelper.GetServicesAsync(baseUrl, _httpClient, _logger) .GetAwaiter().GetResult(); if (services.Count == 0) { _logger.Warn("No services available"); return chain; } HashSet userCountries = _settings.CountryCode .Split([';', ','], StringSplitOptions.RemoveEmptyEntries) .Select(c => c.Trim().ToUpperInvariant()) .ToHashSet(); Dictionary displayToKey = LucidaServiceHelper .ServiceQualityMap.Keys.ToDictionary(k => LucidaServiceHelper.GetServiceDisplayName(k), StringComparer.OrdinalIgnoreCase); IOrderedEnumerable<(string Service, int Priority)> prioritized = _settings.ServicePriorities .Select(kv => (DisplayName: kv.Key, Priority: int.TryParse(kv.Value, out int p) ? p : int.MaxValue)) .Where(x => displayToKey.TryGetValue(x.DisplayName, out _)) .Select(x => (Service: displayToKey[x.DisplayName], x.Priority)) .OrderBy(x => x.Priority); foreach ((string service, int _) in prioritized) { if (!services.TryGetValue(service, out List? countries) || countries.Count == 0) { _logger.Trace("Skipping service {Service}, no countries available", service); continue; } ServiceCountry? preferredCountry = countries.FirstOrDefault(c => userCountries.Contains(c.Code)); string countryCode = preferredCountry?.Code ?? countries[0].Code; string url = $"{baseUrl}/search?query={Uri.EscapeDataString(query)}&service={service}&country={countryCode}"; _logger.Trace("Adding tier: {Url}", url); HttpRequest req = new(url) { RequestTimeout = TimeSpan.FromSeconds(_settings.RequestTimeout), ContentSummary = new LucidaRequestData(service, _settings.BaseUrl, countryCode, isSingle).ToJson() }; req.Headers["User-Agent"] = Tubifarry.UserAgent; chain.AddTier([new IndexerRequest(req)]); } return chain; } } } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaRequestParser.cs ================================================ using Jint; using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Text.Json; using System.Text.RegularExpressions; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Lucida { public interface ILucidaParser : IParseIndexerResponse { } public partial class LucidaParser(Logger logger) : ILucidaParser { private readonly Logger _logger = logger; private static readonly Regex[] SearchDataPatterns = [Data1Regex(), Data2Regex()]; public IList ParseResponse(IndexerResponse indexerResponse) { List releases = []; LucidaRequestData? requestData = GetRequestData(indexerResponse); if (requestData == null) return releases; try { (List? albums, List? tracks) = ExtractSearchResults(indexerResponse.Content); ProcessResults(albums, tracks, releases, requestData); } catch (Exception ex) { _logger.Error(ex, "Error parsing Lucida search response"); } return releases; } private LucidaRequestData? GetRequestData(IndexerResponse indexerResponse) { try { return JsonSerializer.Deserialize(indexerResponse.Request.HttpRequest.ContentSummary ?? string.Empty); } catch (Exception ex) { _logger.Error(ex, "Failed to deserialize request data"); return null; } } private (List? Albums, List? Tracks) ExtractSearchResults(string html) { foreach (Regex pattern in SearchDataPatterns) { Match match = pattern.Match(html); if (!match.Success) continue; try { string raw = NormalizeJsonData(match.Groups[1].Value); List? wrapperList = JsonSerializer.Deserialize>(raw, IndexerParserHelper.StandardJsonOptions); if (wrapperList != null) { LucidaDataWrapper? dataWrapper = wrapperList .FirstOrDefault(w => w.Type == "data" && w.Data?.Results?.Success == true); if (dataWrapper?.Data?.Results?.Results != null) { LucidaResultsData results = dataWrapper.Data.Results.Results; return (results.Albums, results.Tracks); } } } catch (Exception ex) { _logger.Debug(ex, "Typed deserialization failed, trying Jint fallback"); try { return ExtractWithJintToRecords(match.Groups[1].Value); } catch (Exception jintEx) { _logger.Error(jintEx, "Jint extraction failed"); } } } return (null, null); } private static (List? Albums, List? Tracks) ExtractWithJintToRecords(string jsData) { Engine engine = new(); engine.Execute($@" var data = {jsData}; // Find the search results in the data array var searchResults = null; for (var i = 0; i < data.length; i++) {{ var item = data[i]; if (item.type === 'data' && item.data && item.data.results && item.data.results.success) {{ searchResults = item.data.results; break; }} }} // Extract separate arrays for albums and tracks var albums = searchResults && searchResults.results && searchResults.results.albums ? searchResults.results.albums : []; var tracks = searchResults && searchResults.results && searchResults.results.tracks ? searchResults.results.tracks : []; "); object? albumsObj = engine.GetValue("albums").ToObject(); object? tracksObj = engine.GetValue("tracks").ToObject(); string albumsJson = JsonSerializer.Serialize(albumsObj); string tracksJson = JsonSerializer.Serialize(tracksObj); List? albums = null; List? tracks = null; NzbDroneLogger.GetLogger(nameof(LucidaParser)).Info(albumsJson); NzbDroneLogger.GetLogger(nameof(LucidaParser)).Info(tracksJson); if (!string.IsNullOrEmpty(albumsJson) && albumsJson != "[]") albums = JsonSerializer.Deserialize>(albumsJson, IndexerParserHelper.StandardJsonOptions); if (!string.IsNullOrEmpty(tracksJson) && tracksJson != "[]") tracks = JsonSerializer.Deserialize>(tracksJson, IndexerParserHelper.StandardJsonOptions); return (albums, tracks); } private void ProcessResults(List? albums, List? tracks, List releases, LucidaRequestData requestData) { (AudioFormat format, int bitrate, int bitDepth) = LucidaServiceHelper.GetServiceQuality(requestData.ServiceValue); if (albums?.Count > 0) { foreach (LucidaAlbum alb in albums) TryAdd(() => CreateAlbumData(alb, requestData, format, bitrate, bitDepth), releases, alb.Title); } if (tracks != null && requestData.IsSingle && tracks.Count > 0) { foreach (LucidaTrack trk in tracks) TryAdd(() => CreateTrackData(trk, requestData, format, bitrate, bitDepth), releases, trk.Title); } } private void TryAdd(Func factory, List list, string title) { try { list.Add(factory().ToReleaseInfo()); } catch (Exception ex) { _logger.Error(ex, $"Error processing item: {title}"); } } private AlbumData CreateAlbumData(LucidaAlbum album, LucidaRequestData rd, AudioFormat format, int bitrate, int bitDepth) { List artists = album.Artists ?? []; string artist = artists.FirstOrDefault()?.Name ?? "Unknown Artist"; AlbumData data = new("Lucida", nameof(LucidaDownloadProtocol)) { AlbumId = album.Url, AlbumName = album.Title, ArtistName = artist, InfoUrl = $"{rd.BaseUrl}/?url={album.Url}", TotalTracks = album.TrackCount == 0 ? 10 : (int)album.TrackCount, CustomString = "album", Codec = format, Bitrate = bitrate, BitDepth = bitDepth }; ProcessReleaseDate(data, album.ReleaseDate); return data; } private AlbumData CreateTrackData(LucidaTrack track, LucidaRequestData rd, AudioFormat format, int bitrate, int bitDepth) { List artists = track.Artists ?? []; string artist = artists.FirstOrDefault()?.Name ?? "Unknown Artist"; string resolution = string.Empty; AlbumData data = new("Lucida", nameof(LucidaDownloadProtocol)) { AlbumId = track.Url, AlbumName = track.Title, ArtistName = artist, InfoUrl = $"{rd.BaseUrl}/?url={track.Url}", TotalTracks = 1, CustomString = "track", Codec = format, Bitrate = bitrate, BitDepth = bitDepth }; ProcessReleaseDate(data, track.ReleaseDate); return data; } private static void ProcessReleaseDate(AlbumData albumData, string? releaseDate) { if (string.IsNullOrEmpty(releaseDate)) { albumData.ReleaseDate = DateTime.Now.Year.ToString(); albumData.ReleaseDatePrecision = "year"; } else if (ReleaseDateDayRegex().IsMatch(releaseDate)) { albumData.ReleaseDate = releaseDate; albumData.ReleaseDatePrecision = "day"; } else if (ReleaseDateYearRegex().IsMatch(releaseDate)) { albumData.ReleaseDate = releaseDate; albumData.ReleaseDatePrecision = "year"; } else { Match match = ReleaseDateYear2Regex().Match(releaseDate); albumData.ReleaseDate = match.Success ? match.Groups[1].Value : DateTime.Now.Year.ToString(); albumData.ReleaseDatePrecision = "year"; } albumData.ParseReleaseDate(); } private static string NormalizeJsonData(string js) { js = Regex.Replace(js, @"([\{,])\s*([a-zA-Z0-9_$]+)\s*:", "$1\"$2\":"); js = Regex.Replace(js, @":\s*'([^']*)'", ":\"$1\""); js = Regex.Replace(js, @":\s*True\b", ":true"); js = Regex.Replace(js, @":\s*False\b", ":false"); return js; } [GeneratedRegex("^\\d{4}-\\d{2}-\\d{2}$")] private static partial Regex ReleaseDateDayRegex(); [GeneratedRegex("^\\d{4}$")] private static partial Regex ReleaseDateYearRegex(); [GeneratedRegex("\\b(\\d{4})\\b")] private static partial Regex ReleaseDateYear2Regex(); [GeneratedRegex(@"data\s*=\s*(\[(?:[^\[\]]|\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\])*\]);", RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex Data1Regex(); [GeneratedRegex(@"__INITIAL_DATA__\s*=\s*({.+?});", RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex Data2Regex(); } } ================================================ FILE: Tubifarry/Indexers/Lucida/LucidaServiceHelper.cs ================================================ using NLog; using NzbDrone.Common.Http; using Requests; using System.Collections.Concurrent; using System.Net; using System.Text.Json; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Lucida { /// /// Helper class to discover and cache available services from Lucida instances /// public static class LucidaServiceHelper { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; private static readonly ConcurrentDictionary>>> _cache = new(StringComparer.OrdinalIgnoreCase); // Known services that Lucida supports private static readonly IReadOnlyDictionary _knownServices = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["qobuz"] = "Qobuz", ["tidal"] = "Tidal", ["soundcloud"] = "SoundCloud", ["deezer"] = "Deezer", ["amazon"] = "Amazon Music", ["yandex"] = "Yandex Music" }; // Quality mapping for Lucida services public static readonly IReadOnlyDictionary ServiceQualityMap = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["qobuz"] = (AudioFormat.FLAC, 1000, 16), ["tidal"] = (AudioFormat.FLAC, 1000, 16), ["deezer"] = (AudioFormat.MP3, 320, 0), ["soundcloud"] = (AudioFormat.AAC, 128, 0), ["amazon"] = (AudioFormat.FLAC, 1000, 8), ["yandex"] = (AudioFormat.MP3, 320, 0) }; /// /// Gets services available for a specific Lucida instance /// public static Task>> GetServicesAsync( string baseUrl, IHttpClient httpClient, Logger logger) { baseUrl = baseUrl.TrimEnd('/'); return _cache.GetOrAdd(baseUrl, _ => FetchServicesAsync(baseUrl, httpClient, logger)); } /// /// Check if services are available for a specific Lucida instance /// public static bool HasAvailableServices(string baseUrl) { string key = baseUrl.TrimEnd('/'); if (_cache.TryGetValue(key, out Task>>? task) && task.Status == TaskStatus.RanToCompletion) { return task.GetAwaiter().GetResult().Count != 0; } return false; } /// /// Get available services for a specific Lucida instance /// public static Dictionary> GetAvailableServices(string baseUrl) { string key = baseUrl.TrimEnd('/'); if (_cache.TryGetValue(key, out Task>>? task) && task.Status == TaskStatus.RanToCompletion) return task.GetAwaiter().GetResult(); return new Dictionary>(StringComparer.OrdinalIgnoreCase); } /// /// Gets the display name for a service value /// public static string GetServiceDisplayName(string serviceValue) => _knownServices.TryGetValue(serviceValue, out string? display) ? display : char.ToUpperInvariant(serviceValue[0]) + serviceValue[1..]; /// /// Gets the service key for a display name /// public static string? GetServiceKey(string displayName) => _knownServices.FirstOrDefault(kvp => string.Equals(kvp.Value, displayName, StringComparison.OrdinalIgnoreCase)).Key; /// /// Gets the quality information for a service /// public static (AudioFormat Format, int Bitrate, int BitDepth) GetServiceQuality(string serviceValue) => ServiceQualityMap.TryGetValue(serviceValue, out (AudioFormat Format, int Bitrate, int BitDepth) quality) ? quality : (AudioFormat.MP3, 320, 0); /// /// Clear the cached services for a specific instance /// public static void ClearCache(string baseUrl) => _cache.TryRemove(baseUrl.TrimEnd('/'), out _); /// /// Clear all cached services /// public static void ClearAllCaches() => _cache.Clear(); private static async Task>> FetchServicesAsync(string baseUrl, IHttpClient httpClient, Logger logger) { Dictionary> result = new(StringComparer.OrdinalIgnoreCase); RequestContainer container = []; foreach (string service in _knownServices.Keys) { container.Add(new OwnRequest(async _ => { string url = $"{baseUrl}/api/load?url=%2Fapi%2Fcountries%3Fservice%3D{service}"; logger.Trace("Fetching countries for service {Service}: {Url}", service, url); try { HttpRequest req = new(url); req.Headers["User-Agent"] = Tubifarry.UserAgent; HttpResponse response = await httpClient.ExecuteAsync(req); if (response.StatusCode != HttpStatusCode.OK) { logger.Warn("Failed to get countries for service {Service}: {StatusCode}", service, response.StatusCode); return true; } CountryResponse? payload = JsonSerializer.Deserialize(response.Content, _jsonOptions); if (payload?.Success == true && payload.Countries?.Count > 0) { result[service] = payload.Countries; logger.Trace("Found {Count} countries for service {Service}", payload.Countries.Count, service); } } catch (Exception ex) { logger.Error(ex, "Error fetching countries for service {Service}", service); } return true; })); } await container.Task; return result; } } } ================================================ FILE: Tubifarry/Indexers/Soulseek/ISlskdItemsParser.cs ================================================ using Tubifarry.Core.Model; namespace Tubifarry.Indexers.Soulseek { public interface ISlskdItemsParser { SlskdFolderData ParseFolderName(string folderPath); AlbumData CreateAlbumData(string searchId, IGrouping directory, SlskdSearchData searchData, SlskdFolderData folderData, SlskdSettings? settings = null, int expectedTrackCount = 0); } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Core/ISearchStrategy.cs ================================================ namespace Tubifarry.Indexers.Soulseek.Search.Core; public interface ISearchStrategy { string Name { get; } SearchTier Tier { get; } int Priority { get; } bool IsEnabled(SlskdSettings settings); bool CanExecute(SearchContext context, QueryType queryType); string? GetQuery(SearchContext context, QueryType queryType); } public abstract class SearchStrategyBase : ISearchStrategy { public abstract string Name { get; } public abstract SearchTier Tier { get; } public virtual int Priority => 0; public virtual bool IsEnabled(SlskdSettings settings) => true; public abstract bool CanExecute(SearchContext context, QueryType queryType); public abstract string? GetQuery(SearchContext context, QueryType queryType); } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Core/QueryAnalyzer.cs ================================================ using System.Text.RegularExpressions; namespace Tubifarry.Indexers.Soulseek.Search.Core; public static partial class QueryAnalyzer { private static readonly HashSet VariousArtistsNames = new(StringComparer.OrdinalIgnoreCase) { "Various Artists", "VA", "V.A.", "V/A", "Various", "Soundtrack", "OST", "Original Soundtrack", "Compilation", "Mixed By", "DJ Mix" }; private const int SelfTitledFuzzyThreshold = 90; private const int ShortNameThreshold = 4; public static QueryType Analyze(SearchContext context) { QueryType type = QueryType.Normal; if (IsVariousArtists(context.Artist)) type |= QueryType.VariousArtists; if (IsSelfTitled(context.Artist, context.Album)) type |= QueryType.SelfTitled; if (IsShortName(context.Album)) type |= QueryType.ShortName; if (NeedsTypeDisambiguation(context)) type |= QueryType.NeedsTypeDisambiguation; if (HasVolumeReference(context.Album)) type |= QueryType.HasVolume; if (HasStandaloneRomanNumeral(context.Album)) type |= QueryType.HasRomanNumeral; if (NeedsNormalization(context.Artist, context.Album)) type |= QueryType.NeedsNormalization; return type; } public static bool IsVariousArtists(string? artist) { if (string.IsNullOrWhiteSpace(artist)) return false; return VariousArtistsNames.Contains(artist.Trim()); } public static bool IsSelfTitled(string? artist, string? album) { if (string.IsNullOrWhiteSpace(artist) || string.IsNullOrWhiteSpace(album)) return false; string normArtist = NormalizeName(artist); string normAlbum = NormalizeName(album); if (normArtist.Equals(normAlbum, StringComparison.OrdinalIgnoreCase)) return true; // Use token set ratio for better handling of word order differences int similarity = FuzzySharp.Fuzz.TokenSetRatio(normAlbum, normArtist); if (similarity >= SelfTitledFuzzyThreshold) return true; // Check if one fully contains the other (for cases like "Weezer" / "Weezer (Blue Album)") if (normArtist.Length >= 3 && normAlbum.Length >= 3) { string shorter = normArtist.Length < normAlbum.Length ? normArtist : normAlbum; string longer = normArtist.Length < normAlbum.Length ? normAlbum : normArtist; if (longer.StartsWith(shorter, StringComparison.OrdinalIgnoreCase)) return true; } return false; } public static bool IsShortName(string? album) { if (string.IsNullOrWhiteSpace(album)) return false; return album.Trim().Length < ShortNameThreshold; } public static bool NeedsTypeDisambiguation(SearchContext context) { if (context.PrimaryType != NzbDrone.Core.Music.PrimaryAlbumType.EP && context.PrimaryType != NzbDrone.Core.Music.PrimaryAlbumType.Single) return false; // Short names always need disambiguation if (IsShortName(context.Album)) return true; // Self-titled EPs/Singles need disambiguation if (IsSelfTitled(context.Artist, context.Album)) return true; // Common album-like names that could conflict string? album = context.Album?.Trim(); if (string.IsNullOrEmpty(album)) return true; // Single-word titles are ambiguous if (!album.Contains(' ')) return true; return false; } public static bool HasVolumeReference(string? album) => !string.IsNullOrWhiteSpace(album) && VolumeRegex().IsMatch(album); public static bool HasStandaloneRomanNumeral(string? album) { if (string.IsNullOrWhiteSpace(album)) return false; Match romanMatch = StandaloneRomanRegex().Match(album); if (!romanMatch.Success) return false; Match volumeMatch = VolumeRegex().Match(album); if (volumeMatch.Success && volumeMatch.Index <= romanMatch.Index && romanMatch.Index + romanMatch.Length <= volumeMatch.Index + volumeMatch.Length) return false; return true; } public static bool NeedsNormalization(string? artist, string? album) => HasSpecialCharacters(artist) || HasSpecialCharacters(album) || HasPunctuation(artist) || HasPunctuation(album); private static bool HasSpecialCharacters(string? text) => !string.IsNullOrEmpty(text) && SpecialCharRegex().IsMatch(text); private static bool HasPunctuation(string? text) => !string.IsNullOrEmpty(text) && PunctuationRegex().IsMatch(text); private static string NormalizeName(string name) { string normalized = ArticleRegex().Replace(name, ""); normalized = PunctuationRegex().Replace(normalized, ""); return CollapseWhitespaceRegex().Replace(normalized, " ").Trim(); } [GeneratedRegex(@"\b(the|a|an)\s+", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex ArticleRegex(); [GeneratedRegex(@"\s+")] private static partial Regex CollapseWhitespaceRegex(); [GeneratedRegex(@"[^\w\s]", RegexOptions.Compiled)] private static partial Regex PunctuationRegex(); [GeneratedRegex(@"[àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ]", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex SpecialCharRegex(); [GeneratedRegex(@"\b([IVXLCDM]{1,4})\b", RegexOptions.Compiled)] private static partial Regex StandaloneRomanRegex(); [GeneratedRegex(@"\b(?:Vol(?:ume)?\.?)\s*([0-9]+|[IVXLCDM]+)\b", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex VolumeRegex(); } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Core/SearchContext.cs ================================================ using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Music; namespace Tubifarry.Indexers.Soulseek.Search.Core; [Flags] public enum QueryType { Normal = 0, SelfTitled = 1 << 0, ShortName = 1 << 1, VariousArtists = 1 << 2, HasVolume = 1 << 3, HasRomanNumeral = 1 << 4, NeedsNormalization = 1 << 5, NeedsTypeDisambiguation = 1 << 6 } public sealed record SearchContext { public string? Artist { get; init; } public string? Album { get; init; } public string? Year { get; init; } public bool Interactive { get; init; } public int TrackCount { get; init; } public PrimaryAlbumType PrimaryType { get; init; } public IReadOnlyList Aliases { get; init; } public IReadOnlyList Tracks { get; init; } public SlskdSettings Settings { get; init; } public HashSet ProcessedSearches { get; init; } public SearchCriteriaBase? SearchCriteria { get; init; } public QueryType QueryType { get; init; } = QueryType.Normal; public string? NormalizedArtist { get; init; } public string? NormalizedAlbum { get; init; } public string? SearchArtist => IsVariousArtists ? null : (NormalizedArtist ?? Artist); public string? SearchAlbum => NormalizedAlbum ?? Album; public bool IsVariousArtists => QueryType.HasFlag(QueryType.VariousArtists); public bool IsSelfTitled => QueryType.HasFlag(QueryType.SelfTitled); public bool IsShortName => QueryType.HasFlag(QueryType.ShortName); public bool HasValidYear => !string.IsNullOrEmpty(Year) && Year != "0"; public bool NeedsTypeDisambiguation => QueryType.HasFlag(QueryType.NeedsTypeDisambiguation); public string? ReleaseTypeTag => PrimaryType switch { var t when t == PrimaryAlbumType.EP => "EP", var t when t == PrimaryAlbumType.Single => "Single", _ => null }; public SearchContext( string? Artist, string? Album, string? Year, bool Interactive, int TrackCount, PrimaryAlbumType PrimaryType, IReadOnlyList Aliases, IReadOnlyList Tracks, SlskdSettings Settings, HashSet ProcessedSearches, SearchCriteriaBase? SearchCriteria = null) { this.Artist = Artist; this.Album = Album; this.Year = Year; this.Interactive = Interactive; this.TrackCount = TrackCount; this.PrimaryType = PrimaryType; this.Aliases = Aliases; this.Tracks = Tracks; this.Settings = Settings; this.ProcessedSearches = ProcessedSearches; this.SearchCriteria = SearchCriteria; } } public sealed record SearchQuery { public string? Artist { get; init; } public string? Album { get; init; } public bool Interactive { get; init; } public bool ExpandDirectory { get; init; } public int TrackCount { get; init; } public IReadOnlyList Tracks { get; init; } = []; public string? SearchText { get; init; } public static SearchQuery FromContext(SearchContext context) => new() { Artist = context.SearchArtist, Album = context.SearchAlbum, Interactive = context.Interactive, ExpandDirectory = false, TrackCount = context.TrackCount, Tracks = context.Tracks, SearchText = null }; } public delegate IEnumerable SearchExecutor(SearchQuery query); public enum SearchTier { Special = 0, Base = 1, Variation = 2, Fallback = 3 } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Core/SearchPipeline.cs ================================================ using NLog; using NzbDrone.Core.Indexers; using Tubifarry.Core.Utilities; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Core; public interface ISlskdSearchChain { LazyIndexerPageableRequestChain BuildChain(SearchContext context, SearchExecutor searchExecutor); } public sealed class SearchPipeline : ISlskdSearchChain { private readonly IReadOnlyList _strategies; private readonly Logger _logger; public SearchPipeline(IEnumerable strategies, Logger logger) { _logger = logger; _strategies = strategies .OrderBy(s => s.Tier) .ThenBy(s => s.Priority) .ToList() .AsReadOnly(); _logger.Debug($"SearchPipeline: {_strategies.Count} strategies loaded"); } public LazyIndexerPageableRequestChain BuildChain(SearchContext context, SearchExecutor searchExecutor) { var chain = new LazyIndexerPageableRequestChain(context.Settings.MinimumResults); // Analyze and normalize once QueryType queryType = QueryAnalyzer.Analyze(context); SearchContext ctx = ApplyNormalization(context, queryType); _logger.Debug($"Search: Artist='{ctx.Artist}', Album='{ctx.Album}', Type={queryType}"); bool isFirst = true; foreach (var strategy in _strategies) { if (!strategy.IsEnabled(ctx.Settings) || !strategy.CanExecute(ctx, queryType)) continue; Func> factory = () => ExecuteStrategy(strategy, ctx, queryType, searchExecutor); if (isFirst) { chain.AddFactory(factory); isFirst = false; } else { chain.AddTierFactory(factory); } } return chain; } private SearchContext ApplyNormalization(SearchContext context, QueryType queryType) { if (!queryType.HasFlag(QueryType.NeedsNormalization) || !context.Settings.NormalizedSeach) return context with { QueryType = queryType }; var normalized = QueryNormalizer.Normalize(context with { QueryType = queryType }); if (normalized.NormalizedArtist != null || normalized.NormalizedAlbum != null) _logger.Trace($"Normalized: '{normalized.NormalizedArtist ?? context.Artist}' / '{normalized.NormalizedAlbum ?? context.Album}'"); return normalized; } private IEnumerable ExecuteStrategy( ISearchStrategy strategy, SearchContext context, QueryType queryType, SearchExecutor searchExecutor) { string? query = strategy.GetQuery(context, queryType); if (string.IsNullOrWhiteSpace(query)) return []; if (context.ProcessedSearches.Contains(query)) { _logger.Trace($"[{strategy.Name}] Skip duplicate: '{query}'"); return []; } context.ProcessedSearches.Add(query); _logger.Debug($"[{strategy.Name}] Search: '{query}'"); try { var searchQuery = SearchQuery.FromContext(context) with { SearchText = query }; return searchExecutor(searchQuery).ToList(); } catch (Exception ex) { _logger.Error(ex, $"[{strategy.Name}] Error: '{query}'"); return []; } } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Strategies/BaseSearchStrategy.cs ================================================ using Tubifarry.Indexers.Soulseek.Search.Core; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Strategies; public sealed class BaseSearchStrategy : SearchStrategyBase { public override string Name => "Base Search"; public override SearchTier Tier => SearchTier.Base; public override int Priority => 0; public override bool CanExecute(SearchContext context, QueryType queryType) { // Skip if special case already handled it if (queryType.HasFlag(QueryType.VariousArtists) || queryType.HasFlag(QueryType.SelfTitled) || queryType.HasFlag(QueryType.ShortName)) return false; return !string.IsNullOrWhiteSpace(context.SearchArtist) || !string.IsNullOrWhiteSpace(context.SearchAlbum); } // Build: Artist + Album + Year (if valid) + Type (if needed) public override string? GetQuery(SearchContext context, QueryType queryType) => QueryBuilder.Build( context.SearchArtist, context.SearchAlbum, context.Settings.AppendYear && context.HasValidYear ? context.Year : null, context.NeedsTypeDisambiguation ? context.ReleaseTypeTag : null); } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Strategies/FallbackStrategy.cs ================================================ using Tubifarry.Indexers.Soulseek.Search.Core; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Strategies; public sealed class WildcardStrategy : SearchStrategyBase { public override string Name => "Trimmed Fallback"; public override SearchTier Tier => SearchTier.Fallback; public override int Priority => 0; public override bool IsEnabled(SlskdSettings settings) => settings.UseFallbackSearch; public override bool CanExecute(SearchContext context, QueryType queryType) { bool hasArtist = !string.IsNullOrWhiteSpace(context.SearchArtist) && context.SearchArtist.Length > 3; bool hasAlbum = !string.IsNullOrWhiteSpace(context.SearchAlbum) && context.SearchAlbum.Length > 3; return hasArtist || hasAlbum; } public override string? GetQuery(SearchContext context, QueryType queryType) { string artistWildcard = QueryBuilder.BuildTrimmed(context.SearchArtist); if (context.IsSelfTitled) { if (string.IsNullOrWhiteSpace(artistWildcard)) return null; return context.HasValidYear ? QueryBuilder.Build(artistWildcard, context.Year) : artistWildcard; } string albumWildcard = QueryBuilder.BuildTrimmed(context.SearchAlbum); return QueryBuilder.Build(artistWildcard, albumWildcard); } } public sealed class PartialAlbumStrategy : SearchStrategyBase { public override string Name => "Partial Album"; public override SearchTier Tier => SearchTier.Fallback; public override int Priority => 10; public override bool IsEnabled(SlskdSettings settings) => settings.UseFallbackSearch; public override bool CanExecute(SearchContext context, QueryType queryType) => !context.IsSelfTitled && !string.IsNullOrWhiteSpace(context.SearchAlbum) && context.SearchAlbum.Length >= 15; public override string? GetQuery(SearchContext context, QueryType queryType) { string? partial = QueryBuilder.BuildPartial(context.SearchAlbum); if (string.IsNullOrWhiteSpace(partial)) return null; return QueryBuilder.Build(context.SearchArtist, partial); } } public sealed class AliasStrategy : SearchStrategyBase { private const int MinAliasLength = 4; public override string Name => "Artist Alias"; public override SearchTier Tier => SearchTier.Fallback; public override int Priority => 20; public override bool IsEnabled(SlskdSettings settings) => settings.UseFallbackSearch; public override bool CanExecute(SearchContext context, QueryType queryType) => !context.IsVariousArtists && context.Aliases.Count > 0 && context.Aliases.Any(a => !string.IsNullOrWhiteSpace(a) && a.Length >= MinAliasLength); public override string? GetQuery(SearchContext context, QueryType queryType) { string? alias = context.Aliases .FirstOrDefault(a => !string.IsNullOrWhiteSpace(a) && a.Length >= MinAliasLength && !a.Equals(context.Artist, StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrWhiteSpace(alias)) return null; return QueryBuilder.Build(alias, context.SearchAlbum); } } public sealed class TrackFallbackStrategy : SearchStrategyBase { private const int MinTrackLength = 5; public override string Name => "Track Fallback"; public override SearchTier Tier => SearchTier.Fallback; public override int Priority => 30; public override bool IsEnabled(SlskdSettings settings) => settings.UseTrackFallback; public override bool CanExecute(SearchContext context, QueryType queryType) => context.Tracks.Count > 0 && context.Tracks.Any(t => !string.IsNullOrWhiteSpace(t) && t.Trim().Length >= MinTrackLength); public override string? GetQuery(SearchContext context, QueryType queryType) { // Find most distinctive track (longer, fewer common words) string? track = context.Tracks .Where(t => !string.IsNullOrWhiteSpace(t) && t.Trim().Length >= MinTrackLength) .OrderByDescending(t => t.Length) .FirstOrDefault()? .Trim(); if (string.IsNullOrWhiteSpace(track)) return null; return QueryBuilder.Build(context.SearchArtist, track); } } public sealed class DistinctiveAlbumStrategy : SearchStrategyBase { public override string Name => "Distinctive Album"; public override SearchTier Tier => SearchTier.Fallback; public override int Priority => 15; public override bool IsEnabled(SlskdSettings settings) => settings.UseFallbackSearch; public override bool CanExecute(SearchContext context, QueryType queryType) => !context.IsSelfTitled && !string.IsNullOrWhiteSpace(context.SearchAlbum) && context.SearchAlbum.Length >= 10; public override string? GetQuery(SearchContext context, QueryType queryType) { string distinctive = QueryBuilder.ExtractDistinctive(context.SearchAlbum); if (string.IsNullOrWhiteSpace(distinctive) || distinctive.Equals(context.SearchAlbum, StringComparison.OrdinalIgnoreCase)) return null; return QueryBuilder.Build(context.SearchArtist, distinctive); } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Strategies/SpecialCaseStrategy.cs ================================================ using Tubifarry.Indexers.Soulseek.Search.Core; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Strategies; public sealed class VariousArtistsStrategy : SearchStrategyBase { public override string Name => "Various Artists"; public override SearchTier Tier => SearchTier.Special; public override int Priority => 10; public override bool CanExecute(SearchContext context, QueryType queryType) => queryType.HasFlag(QueryType.VariousArtists) && !string.IsNullOrWhiteSpace(context.SearchAlbum); public override string? GetQuery(SearchContext context, QueryType queryType) { if (context.HasValidYear) return QueryBuilder.Build(context.SearchAlbum, context.Year, context.ReleaseTypeTag); if (context.NeedsTypeDisambiguation) return QueryBuilder.Build(context.SearchAlbum, context.ReleaseTypeTag); return context.SearchAlbum; } } public sealed class SelfTitledStrategy : SearchStrategyBase { public override string Name => "Self-Titled"; public override SearchTier Tier => SearchTier.Special; public override int Priority => 20; public override bool CanExecute(SearchContext context, QueryType queryType) => queryType.HasFlag(QueryType.SelfTitled) && !queryType.HasFlag(QueryType.VariousArtists) && !string.IsNullOrWhiteSpace(context.SearchArtist); public override string? GetQuery(SearchContext context, QueryType queryType) { if (context.HasValidYear) return QueryBuilder.Build(context.SearchArtist, context.Year, context.ReleaseTypeTag); if (context.NeedsTypeDisambiguation) return QueryBuilder.Build(context.SearchArtist, context.ReleaseTypeTag); return context.SearchArtist; } } public sealed class ShortNameStrategy : SearchStrategyBase { public override string Name => "Short Name"; public override SearchTier Tier => SearchTier.Special; public override int Priority => 30; public override bool CanExecute(SearchContext context, QueryType queryType) => queryType.HasFlag(QueryType.ShortName) && !queryType.HasFlag(QueryType.VariousArtists) && !queryType.HasFlag(QueryType.SelfTitled); public override string? GetQuery(SearchContext context, QueryType queryType) { // Short album names need maximum context return QueryBuilder.Build( context.SearchArtist, context.SearchAlbum, context.HasValidYear ? context.Year : null, context.ReleaseTypeTag); } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Strategies/TemplateSearchStrategy.cs ================================================ using Tubifarry.Indexers.Soulseek.Search.Core; using Tubifarry.Indexers.Soulseek.Search.Templates; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Strategies; /// /// Search strategy that uses user-configured templates with reflection-based placeholder resolution. /// Placeholders use {{Property}} syntax and can access nested properties up to 3 levels deep. /// Examples: {{ArtistQuery}}, {{AlbumTitle}}, {{Artist.Metadata.Value.Aliases[0]}} /// public sealed class TemplateSearchStrategy : SearchStrategyBase { public override string Name => "Template"; public override SearchTier Tier => SearchTier.Special; public override int Priority => 0; public override bool IsEnabled(SlskdSettings settings) => !string.IsNullOrWhiteSpace(settings.SearchTemplates); public override bool CanExecute(SearchContext context, QueryType queryType) => context.SearchCriteria != null; public override string? GetQuery(SearchContext context, QueryType queryType) { IReadOnlyList templates = TemplateEngine.ParseTemplates(context.Settings.SearchTemplates); foreach (string template in templates) { string? result = TemplateEngine.Apply(template, context.SearchCriteria); if (!string.IsNullOrWhiteSpace(result)) { result = QueryBuilder.DeduplicateTerms(result); return string.IsNullOrWhiteSpace(result) ? null : result; } } return null; } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Strategies/VariationStrategy.cs ================================================ using Tubifarry.Indexers.Soulseek.Search.Core; using Tubifarry.Indexers.Soulseek.Search.Transformers; namespace Tubifarry.Indexers.Soulseek.Search.Strategies; public sealed class VolumeVariationStrategy : SearchStrategyBase { public override string Name => "Volume Variation"; public override SearchTier Tier => SearchTier.Variation; public override int Priority => 0; public override bool IsEnabled(SlskdSettings settings) => settings.HandleVolumeVariations; public override bool CanExecute(SearchContext context, QueryType queryType) => queryType.HasFlag(QueryType.HasVolume) && !string.IsNullOrWhiteSpace(context.SearchAlbum); public override string? GetQuery(SearchContext context, QueryType queryType) { string? converted = QueryBuilder.ConvertVolumeFormat(context.SearchAlbum); if (string.IsNullOrWhiteSpace(converted)) return null; return QueryBuilder.Build(context.SearchArtist, converted); } } public sealed class RomanNumeralVariationStrategy : SearchStrategyBase { public override string Name => "Roman Numeral"; public override SearchTier Tier => SearchTier.Variation; public override int Priority => 10; public override bool IsEnabled(SlskdSettings settings) => settings.HandleVolumeVariations; public override bool CanExecute(SearchContext context, QueryType queryType) => queryType.HasFlag(QueryType.HasRomanNumeral) && !string.IsNullOrWhiteSpace(context.SearchAlbum); public override string? GetQuery(SearchContext context, QueryType queryType) { string? converted = QueryBuilder.ConvertRomanNumeral(context.SearchAlbum); if (string.IsNullOrWhiteSpace(converted)) return null; return QueryBuilder.Build(context.SearchArtist, converted); } } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Templates/TemplateEngine.cs ================================================ using NzbDrone.Core.IndexerSearch.Definitions; using System.Collections; using System.Collections.Concurrent; using System.Reflection; using System.Text.RegularExpressions; namespace Tubifarry.Indexers.Soulseek.Search.Templates; public static partial class TemplateEngine { private static readonly Regex PlaceholderRegex = CreatePlaceholderRegex(); private static readonly Regex IndexerRegex = CreateIndexerRegex(); private static readonly Type SearchCriteriaType = typeof(AlbumSearchCriteria); private static readonly ConcurrentDictionary<(Type Type, string Name), PropertyInfo?> PropertyCache = new(); private static readonly ConcurrentDictionary<(Type Type, string Name), FieldInfo?> FieldCache = new(); public static IReadOnlyList ParseTemplates(string? templateConfig) { if (string.IsNullOrWhiteSpace(templateConfig)) return []; return templateConfig .Split(['\n', '\r', ';'], StringSplitOptions.RemoveEmptyEntries) .Select(l => l.Trim()) .Where(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith('#') && !l.StartsWith("//")) .ToList() .AsReadOnly(); } public static IReadOnlyList ValidateTemplates(string? templateConfig) { List errors = new List(); IReadOnlyList templates = ParseTemplates(templateConfig); for (int i = 0; i < templates.Count; i++) { MatchCollection matches = PlaceholderRegex.Matches(templates[i]); if (matches.Count == 0) { errors.Add($"Line {i + 1}: No placeholders found (use {{{{Property}}}})"); continue; } foreach (Match match in matches) { string path = match.Groups[1].Value.Trim(); string? error = ValidatePath(path); if (error != null) errors.Add($"Line {i + 1}: '{path}' - {error}"); } } return errors.AsReadOnly(); } private static string? ValidatePath(string path) { if (string.IsNullOrWhiteSpace(path)) return "empty path"; Type? currentType = SearchCriteriaType; foreach (string segment in path.Split('.').Take(3)) { if (currentType == null) return "cannot resolve deeper"; Match idx = IndexerRegex.Match(segment); string propName = idx.Success ? idx.Groups[1].Value : segment; PropertyInfo? prop = GetCachedProperty(currentType, propName); if (prop == null) return $"unknown property '{propName}'"; currentType = prop.PropertyType; // Unwrap Lazy, List, etc. if (currentType.IsGenericType) { Type genDef = currentType.GetGenericTypeDefinition(); if (genDef == typeof(Lazy<>) || genDef == typeof(List<>) || genDef == typeof(IList<>) || genDef == typeof(IEnumerable<>)) currentType = currentType.GetGenericArguments()[0]; } } return null; } public static string? Apply(string template, object? context) { if (string.IsNullOrWhiteSpace(template) || context == null) return null; bool hasUnresolved = false; string result = PlaceholderRegex.Replace(template, match => { object? value = ResolvePath(context, match.Groups[1].Value.Trim()); if (value == null || (value is string s && string.IsNullOrWhiteSpace(s))) { hasUnresolved = true; return string.Empty; } return value.ToString() ?? string.Empty; }); if (hasUnresolved) return null; result = MultiSpaceRegex().Replace(result, " ").Trim(); return string.IsNullOrWhiteSpace(result) ? null : result; } private static object? ResolvePath(object? obj, string path) { if (obj == null || string.IsNullOrWhiteSpace(path)) return null; try { object? current = obj; foreach (string segment in path.Split('.').Take(3)) { if (current == null) break; Match idx = IndexerRegex.Match(segment); if (idx.Success) { current = GetValue(current, idx.Groups[1].Value); current = GetIndexed(current, int.Parse(idx.Groups[2].Value)); } else { current = GetValue(current, segment); } } return current; } catch { return null; } } private static object? GetValue(object obj, string name) { Type type = obj.GetType(); PropertyInfo? prop = GetCachedProperty(type, name); object? value = prop?.GetValue(obj) ?? GetCachedField(type, name)?.GetValue(obj); // Unwrap Lazy if (value?.GetType() is { IsGenericType: true } t && t.GetGenericTypeDefinition() == typeof(Lazy<>)) value = GetCachedProperty(t, "Value")?.GetValue(value); return value; } private static object? GetIndexed(object? col, int idx) => col switch { null => null, IList list => idx < list.Count ? list[idx] : null, IEnumerable e => e.Cast().ElementAtOrDefault(idx), _ => null }; private static PropertyInfo? GetCachedProperty(Type type, string name) => PropertyCache.GetOrAdd((type, name), key => key.Type.GetProperty(key.Name, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase)); private static FieldInfo? GetCachedField(Type type, string name) => FieldCache.GetOrAdd((type, name), key => key.Type.GetField(key.Name, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase)); public static void ClearCaches() { PropertyCache.Clear(); FieldCache.Clear(); } [GeneratedRegex(@"\{\{([^}]+)\}\}", RegexOptions.Compiled)] private static partial Regex CreatePlaceholderRegex(); [GeneratedRegex(@"^(\w+)\[(\d+)\]$", RegexOptions.Compiled)] private static partial Regex CreateIndexerRegex(); [GeneratedRegex(@"\s{2,}", RegexOptions.Compiled)] private static partial Regex MultiSpaceRegex(); } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Transformers/QueryBuilder.cs ================================================ using System.Text.RegularExpressions; namespace Tubifarry.Indexers.Soulseek.Search.Transformers; public static partial class QueryBuilder { private static readonly HashSet StopWords = new(StringComparer.OrdinalIgnoreCase) { "the", "a", "an", "and", "or", "but", "of", "at", "by", "for", "with", "as", "to", "in", "on", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "from", "into" }; private const int MinWordLengthForTrim = 4; private const int MinAlbumLengthForPartial = 15; private const int MinSignificantWordsForPartial = 2; public static string Build(params string?[] parts) => string.Join(" ", parts.Where(p => !string.IsNullOrWhiteSpace(p))).Trim(); public static string DeduplicateTerms(string? text) { if (string.IsNullOrWhiteSpace(text)) return text ?? string.Empty; string[] words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries); if (words.Length < 2) return text; for (int seqLen = words.Length / 2; seqLen >= 1; seqLen--) { for (int i = 0; i <= words.Length - 2 * seqLen; i++) { bool match = true; for (int j = 0; j < seqLen; j++) { if (!words[i + j].Equals(words[i + seqLen + j], StringComparison.OrdinalIgnoreCase)) { match = false; break; } } if (match) { List result = new List(words.Length - seqLen); result.AddRange(words.Take(i + seqLen)); result.AddRange(words.Skip(i + 2 * seqLen)); return string.Join(" ", result); } } } return text; } public static string BuildTrimmed(string? text) { if (string.IsNullOrWhiteSpace(text)) return string.Empty; string[] words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < words.Length; i++) { if (words[i].Length >= MinWordLengthForTrim && !StopWords.Contains(words[i])) words[i] = words[i][..^1]; } return string.Join(" ", words); } public static string? BuildPartial(string? text) { if (string.IsNullOrWhiteSpace(text) || text.Length < MinAlbumLengthForPartial) return null; string[] words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries); // Get significant words (non-stopwords) List significant = words.Where(w => !StopWords.Contains(w) && w.Length > 1).ToList(); if (significant.Count < MinSignificantWordsForPartial) return null; // Strategy: Take unique/longer words first (more distinctive) List prioritized = significant .OrderByDescending(w => w.Length) .ThenBy(w => w) .Take(Math.Max(MinSignificantWordsForPartial, (significant.Count + 1) / 2)) .ToList(); // Preserve original order List result = words.Where(w => prioritized.Contains(w, StringComparer.OrdinalIgnoreCase)).ToList(); string partial = string.Join(" ", result); // Don't return if it's the same as input or too short if (partial.Equals(text, StringComparison.OrdinalIgnoreCase) || partial.Length < 5) return null; return partial; } public static string ExtractDistinctive(string? text, int maxWords = 3) { if (string.IsNullOrWhiteSpace(text)) return string.Empty; // Remove parenthetical content first (usually metadata like "(Deluxe Edition)") string cleaned = ParenthesesRegex().Replace(text, "").Trim(); if (string.IsNullOrWhiteSpace(cleaned)) cleaned = text; string[] words = cleaned.Split(' ', StringSplitOptions.RemoveEmptyEntries); // Filter and prioritize List candidates = words .Where(w => !StopWords.Contains(w) && w.Length > 2) .OrderByDescending(w => w.Length) .Take(maxWords) .ToList(); if (candidates.Count == 0) return cleaned; // Return in original order return string.Join(" ", words.Where(w => candidates.Contains(w, StringComparer.OrdinalIgnoreCase))); } public static string? ConvertVolumeFormat(string? album) { if (string.IsNullOrWhiteSpace(album)) return null; Match match = VolumeRegex().Match(album); if (!match.Success) return null; string format = match.Groups[1].Value; string number = match.Groups[2].Value; // Convert number format string convertedNumber = ConvertNumber(number); if (convertedNumber == number) { // Number didn't change, try changing the format word string newFormat = format.Contains("ume", StringComparison.OrdinalIgnoreCase) ? "Vol." : "Volume"; return album.Replace(match.Value, $"{newFormat} {number}"); } return album.Replace(match.Value, $"{format} {convertedNumber}"); } public static string? ConvertRomanNumeral(string? album) { if (string.IsNullOrWhiteSpace(album)) return null; Match romanMatch = StandaloneRomanRegex().Match(album); if (!romanMatch.Success) return null; // Skip if part of volume reference Match volumeMatch = VolumeRegex().Match(album); if (volumeMatch.Success && romanMatch.Index >= volumeMatch.Index && romanMatch.Index + romanMatch.Length <= volumeMatch.Index + volumeMatch.Length) return null; string converted = ConvertNumber(romanMatch.Groups[1].Value); if (converted == romanMatch.Groups[1].Value) return null; return album.Replace(romanMatch.Value, converted); } private static readonly Dictionary RomanToArabic = new(StringComparer.OrdinalIgnoreCase) { ["I"] = 1, ["II"] = 2, ["III"] = 3, ["IV"] = 4, ["V"] = 5, ["VI"] = 6, ["VII"] = 7, ["VIII"] = 8, ["IX"] = 9, ["X"] = 10, ["XI"] = 11, ["XII"] = 12, ["XIII"] = 13, ["XIV"] = 14, ["XV"] = 15 }; private static string ConvertNumber(string number) { if (RomanToArabic.TryGetValue(number, out int arabic)) return arabic.ToString(); if (int.TryParse(number, out arabic) && arabic is > 0 and <= 15) { KeyValuePair roman = RomanToArabic.FirstOrDefault(x => x.Value == arabic); if (!string.IsNullOrEmpty(roman.Key)) return roman.Key; } return number; } [GeneratedRegex(@"\([^)]*\)|\[[^\]]*\]", RegexOptions.Compiled)] private static partial Regex ParenthesesRegex(); [GeneratedRegex(@"\b([IVXLCDM]{1,4})\b", RegexOptions.Compiled)] private static partial Regex StandaloneRomanRegex(); [GeneratedRegex(@"\b(Vol(?:ume)?\.?)\s*([0-9]+|[IVXLCDM]+)\b", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex VolumeRegex(); } ================================================ FILE: Tubifarry/Indexers/Soulseek/Search/Transformers/QueryNormalizer.cs ================================================ using System.Globalization; using System.Text; using System.Text.RegularExpressions; using Tubifarry.Indexers.Soulseek.Search.Core; namespace Tubifarry.Indexers.Soulseek.Search.Transformers; public static partial class QueryNormalizer { public static SearchContext Normalize(SearchContext context) { if (!context.QueryType.HasFlag(QueryType.NeedsNormalization)) return context; string? normArtist = NormalizeText(context.Artist); string? normAlbum = NormalizeText(context.Album); bool artistChanged = !string.Equals(normArtist, context.Artist, StringComparison.Ordinal); bool albumChanged = !string.Equals(normAlbum, context.Album, StringComparison.Ordinal); if (!artistChanged && !albumChanged) return context; return context with { NormalizedArtist = artistChanged ? normArtist : null, NormalizedAlbum = albumChanged ? normAlbum : null }; } public static string NormalizeText(string? input) { if (string.IsNullOrEmpty(input)) return input ?? string.Empty; // Decompose accented characters (é → e + ´) string decomposed = input.Normalize(NormalizationForm.FormD); StringBuilder sb = new(decomposed.Length); foreach (char c in decomposed) { UnicodeCategory category = CharUnicodeInfo.GetUnicodeCategory(c); if (category != UnicodeCategory.NonSpacingMark && category != UnicodeCategory.SpacingCombiningMark && category != UnicodeCategory.EnclosingMark) { sb.Append(c); } } string result = sb.ToString().Normalize(NormalizationForm.FormC); // Then strip punctuation but keep letters, digits, spaces, hyphens, ampersands result = PlusRegex().Replace(result, " "); result = PunctuationRegex().Replace(result, ""); result = WhitespaceRegex().Replace(result, " ").Trim(); return result; } [GeneratedRegex(@"[^\w\s\-&]", RegexOptions.Compiled)] private static partial Regex PunctuationRegex(); [GeneratedRegex(@"\+")] private static partial Regex PlusRegex(); [GeneratedRegex(@"\s+")] private static partial Regex WhitespaceRegex(); } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlsdkRecords.cs ================================================ using System.Text.Json; using System.Text.Json.Serialization; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Soulseek { public record SlskdSearchResponse( [property: JsonPropertyName("id")] string Id, [property: JsonPropertyName("searchText")] string SearchText, [property: JsonPropertyName("startedAt")] DateTime StartedAt, [property: JsonPropertyName("endedAt")] DateTime? EndedAt, [property: JsonPropertyName("state")] string State, [property: JsonPropertyName("isComplete")] bool IsComplete, [property: JsonPropertyName("fileCount")] int FileCount, [property: JsonPropertyName("responseCount")] int ResponseCount, [property: JsonPropertyName("token")] int Token, [property: JsonPropertyName("responses")] List Responses ); public record SlskdLockedFile( [property: JsonPropertyName("filename")] string Filename ); public record SlskdFileData( [property: JsonPropertyName("filename")] string? Filename, [property: JsonPropertyName("bitRate")] int? BitRate, [property: JsonPropertyName("bitDepth")] int? BitDepth, [property: JsonPropertyName("size")] long Size, [property: JsonPropertyName("length")] int? Length, [property: JsonPropertyName("extension")] string? Extension, [property: JsonPropertyName("sampleRate")] int? SampleRate, [property: JsonPropertyName("code")] int Code, [property: JsonPropertyName("isLocked")] bool IsLocked) { public static IEnumerable GetFilteredFiles(List files, bool onlyIncludeAudio = false, IEnumerable? includedFileExtensions = null) { foreach (SlskdFileData file in files) { string? extension = !string.IsNullOrWhiteSpace(file.Extension) ? file.Extension : Path.GetExtension(file.Filename); if (onlyIncludeAudio && AudioFormatHelper.GetAudioCodecFromExtension(extension ?? "") == AudioFormat.Unknown && !(includedFileExtensions?.Contains(extension, StringComparer.OrdinalIgnoreCase) ?? false)) { continue; } yield return file with { Extension = extension }; } } } public record SlskdFolderData( string Path, string Artist, string Album, string Year, [property: JsonPropertyName("username")] string Username, [property: JsonPropertyName("hasFreeUploadSlot")] bool HasFreeUploadSlot, [property: JsonPropertyName("uploadSpeed")] long UploadSpeed, [property: JsonPropertyName("lockedFileCount")] int LockedFileCount, [property: JsonPropertyName("lockedFiles")] List LockedFiles, [property: JsonPropertyName("queueLength")] int QueueLength, [property: JsonPropertyName("token")] int Token, [property: JsonPropertyName("fileCount")] int FileCount, [property: JsonPropertyName("files")] List Files) { public int CalculatePriority(int expectedTrackCount = 0) { // Early exit: completely locked folders if (LockedFileCount >= FileCount && FileCount > 0) return 0; // Early exit: more than 50% locked = useless source double availabilityRatio = FileCount > 0 ? (FileCount - LockedFileCount) / (double)FileCount : 1.0; if (availabilityRatio <= 0.5) return 0; int score = 0; // Get actual track count int actualTrackCount = 0; if (expectedTrackCount > 0) { actualTrackCount = Files.Count(f => AudioFormatHelper.GetAudioCodecFromExtension( f.Extension ?? System.IO.Path.GetExtension(f.Filename) ?? "") != AudioFormat.Unknown); } // Early exit: Missing 50%+ of expected tracks if (expectedTrackCount > 0 && actualTrackCount > 0 && actualTrackCount <= expectedTrackCount * 0.5) return 0; // ===== TRACK COUNT MATCHING (0 to +2500) ===== if (expectedTrackCount > 0 && actualTrackCount > 0) { int trackDiff = actualTrackCount - expectedTrackCount; if (trackDiff < 0) // -1: ~500 pts, -2: ~60 pts, -3+: near 0 score += (int)(2500 * Math.Exp(-Math.Pow(Math.Abs(trackDiff), 2) * 5)); else if (trackDiff == 0) // PERFECT MATCH score += 2500; else // EXTRA TRACKS: Less critical, just penalized: +1: ~1600 pts, +2: ~600 pts, +3: ~100 pts, +15: near 0 score += (int)(2500 * Math.Exp(-Math.Pow(trackDiff, 2) * 1.5)); } // ===== AVAILABILITY RATIO (0 to +2000) ===== score += (int)(Math.Pow(availabilityRatio, 2.0) * 2000); // ===== UPLOAD SPEED (0 to +1800) ===== if (UploadSpeed > 0) { double speedMbps = UploadSpeed / (1024.0 * 1024.0 / 8.0); score += Math.Min(1800, (int)(Math.Log10(Math.Max(0.1, speedMbps) + 1) * 1100)); } // ===== QUEUE LENGTH (50 to +1500) ===== double queueFactor = Math.Pow(0.94, Math.Min(QueueLength, 40)); score += (int)(queueFactor * 1500); // ===== FREE UPLOAD SLOT (0 or +800) ===== score += HasFreeUploadSlot ? 800 : 0; // ===== COLLECTION SIZE (0 to +300) ===== score += Math.Min(300, (int)(Math.Log10(Math.Max(1, FileCount) + 1) * 150)); return Math.Clamp(score, 0, 10000); } } public record SlskdSearchData( [property: JsonPropertyName("artist")] string? Artist, [property: JsonPropertyName("album")] string? Album, [property: JsonPropertyName("interactive")] bool Interactive, [property: JsonPropertyName("expandDirectory")] bool ExpandDirectory, [property: JsonPropertyName("mimimumFiles")] int MinimumFiles, [property: JsonPropertyName("maximumFiles")] int? MaximumFiles) { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; public static SlskdSearchData FromJson(string jsonString) => JsonSerializer.Deserialize(jsonString, _jsonOptions)!; } public record SlskdDirectoryApiResponse( [property: JsonPropertyName("files")] List Files ); public record SlskdDirectoryApiFile( [property: JsonPropertyName("filename")] string Filename, [property: JsonPropertyName("size")] long Size, [property: JsonPropertyName("code")] int Code ); } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdIndexer.cs ================================================ using FluentValidation.Results; using Newtonsoft.Json; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download.History; using NzbDrone.Core.History; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Queue; using System.Net; using Tubifarry.Core.Replacements; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; using Tubifarry.Indexers.Soulseek.Search.Core; namespace Tubifarry.Indexers.Soulseek { public class SlskdIndexer : ExtendedHttpIndexerBase { public override string Name => "Slskd"; public override string Protocol => nameof(SoulseekDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 50; public override TimeSpan RateLimit => new(3); private readonly SlskdRequestGenerator _indexerRequestGenerator; private readonly IParseIndexerResponse _parseIndexerResponse; internal new SlskdSettings Settings => base.Settings; public SlskdIndexer(IHttpClient httpClient, Lazy indexerFactory, IIndexerStatusService indexerStatusService, ISlskdSearchChain slskdSearchChain, ISlskdItemsParser slskdItemsParser, IHistoryService historyService, IDownloadHistoryService downloadHistoryService, IQueueService queueService, IConfigService configService, IParsingService parsingService, ISentryHelper sentry, Logger logger) : base(httpClient, indexerStatusService, configService, parsingService, sentry, logger) { _parseIndexerResponse = new SlskdIndexerParser(this, indexerFactory, httpClient, slskdItemsParser, historyService, downloadHistoryService, queueService, sentry); _indexerRequestGenerator = new SlskdRequestGenerator(this, slskdSearchChain, httpClient, sentry); } protected override IList CleanupReleases(IEnumerable releases, bool isRecent = false) { IList result = base.CleanupReleases(releases, isRecent); foreach (ReleaseInfo release in result) { if (release is not TorrentInfo slskd) continue; int basePriority = release.IndexerPriority; int score = Math.Clamp(slskd.Seeders ?? 0, 0, 10000); release.IndexerPriority = basePriority + 12 - (int)Math.Round(score / 10000.0 * 24); } return result; } protected override async Task Test(List failures) => failures.AddIfNotNull(await TestConnection()); public override IIndexerRequestGenerator GetExtendedRequestGenerator() => _indexerRequestGenerator; public override IParseIndexerResponse GetParser() => _parseIndexerResponse; protected override async Task TestConnection() { try { HttpRequest request = new HttpRequestBuilder($"{Settings.BaseUrl}/api/v0/application") .SetHeader("X-API-KEY", Settings.ApiKey).Build(); request.AllowAutoRedirect = true; request.RequestTimeout = TimeSpan.FromSeconds(30); HttpResponse response = await _httpClient.ExecuteAsync(request); _logger.Debug($"TestConnection Response: {response.Content}"); if (response.StatusCode != HttpStatusCode.OK) return new ValidationFailure("BaseUrl", $"Unable to connect to Slskd. Status: {response.StatusCode}"); dynamic? jsonResponse = JsonConvert.DeserializeObject(response.Content); if (jsonResponse == null) return new ValidationFailure("BaseUrl", "Failed to parse Slskd response."); string? serverState = jsonResponse?.server?.state?.ToString(); if (string.IsNullOrEmpty(serverState) || !serverState.Contains("Connected")) return new ValidationFailure("BaseUrl", $"Slskd server is not connected. State: {serverState}"); if (!string.IsNullOrWhiteSpace(Settings.IgnoreListPath)) { if (!File.Exists(Settings.IgnoreListPath)) return new ValidationFailure("IgnoreListPath", "Ignore List File does not exists."); SlskdIndexerParser.InvalidIgnoreCache(Settings.IgnoreListPath); return PermissionTester.TestReadWritePermissions(Path.GetDirectoryName(Settings.IgnoreListPath)!, _logger)!; } return null!; } catch (HttpException ex) { _logger.Warn(ex, "Unable to connect to Slskd."); return new ValidationFailure("BaseUrl", $"Unable to connect to Slskd: {ex.Message}"); } catch (Exception ex) { _logger.Error(ex, "Unexpected error while testing Slskd connection."); return new ValidationFailure(string.Empty, $"Unexpected error: {ex.Message}"); } } } } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdIndexerParser.cs ================================================ using FuzzySharp; using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Download; using NzbDrone.Core.Download.History; using NzbDrone.Core.History; using NzbDrone.Core.Indexers; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Queue; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Soulseek { public class SlskdIndexerParser : IParseIndexerResponse, IHandle, IHandle { private readonly SlskdIndexer _indexer; private readonly Logger _logger; private readonly Lazy _indexerFactory; private readonly IHttpClient _httpClient; private readonly ISlskdItemsParser _itemsParser; private readonly IHistoryService _historyService; private readonly IDownloadHistoryService _downloadHistoryService; private readonly IQueueService _queueService; private readonly ISentryHelper _sentry; private static readonly Dictionary _interactiveResults = []; private static readonly Dictionary IgnoredUsers, long LastFileSize)> _ignoreListCache = new(); private readonly object _rateLimitLock = new(); private DateTime _rateLimitCacheTimestamp = DateTime.MinValue; private HashSet _rateLimitedUsersCache = new(StringComparer.OrdinalIgnoreCase); private SlskdSettings Settings => _indexer.Settings; public SlskdIndexerParser(SlskdIndexer indexer, Lazy indexerFactory, IHttpClient httpClient, ISlskdItemsParser itemsParser, IHistoryService historyService, IDownloadHistoryService downloadHistoryService, IQueueService queueService, ISentryHelper sentry) { _indexer = indexer; _indexerFactory = indexerFactory; _logger = NzbDroneLogger.GetLogger(this); _httpClient = httpClient; _itemsParser = itemsParser; _historyService = historyService; _downloadHistoryService = downloadHistoryService; _queueService = queueService; _sentry = sentry; } public IList ParseResponse(IndexerResponse indexerResponse) { List albumDatas = []; try { SlskdSearchResponse? searchResponse = JsonSerializer.Deserialize(indexerResponse.Content, IndexerParserHelper.StandardJsonOptions); if (searchResponse == null) { _logger.Error("Failed to deserialize slskd search response."); return []; } SlskdSearchData searchTextData = SlskdSearchData.FromJson(indexerResponse.HttpRequest.ContentSummary); HashSet? ignoredUsers = GetIgnoredUsers(Settings.IgnoreListPath); HashSet rateLimitedUsers = GetRateLimitedUsers(); foreach (SlskdFolderData response in searchResponse.Responses) { if (ignoredUsers?.Contains(response.Username) == true || rateLimitedUsers.Contains(response.Username)) continue; IEnumerable filteredFiles = SlskdFileData.GetFilteredFiles(response.Files, Settings.OnlyAudioFiles, Settings.IncludeFileExtensions); foreach (IGrouping directoryGroup in filteredFiles.GroupBy(f => SlskdTextProcessor.GetDirectoryFromFilename(f.Filename))) { if (string.IsNullOrEmpty(directoryGroup.Key)) continue; SlskdFolderData folderData = _itemsParser.ParseFolderName(directoryGroup.Key) with { Username = response.Username, HasFreeUploadSlot = response.HasFreeUploadSlot, UploadSpeed = response.UploadSpeed, LockedFileCount = response.LockedFileCount, LockedFiles = response.LockedFiles, QueueLength = response.QueueLength, Token = response.Token, FileCount = response.FileCount }; IGrouping finalGroup = directoryGroup; if (searchTextData.ExpandDirectory) { IGrouping? expandedGroup = TryExpandDirectory(searchTextData, directoryGroup, folderData); if (expandedGroup != null) finalGroup = expandedGroup; } if (searchTextData.MinimumFiles > 0 || searchTextData.MaximumFiles.HasValue) { bool filterActive = (TrackCountFilterType)Settings.TrackCountFilter != TrackCountFilterType.Disabled; int fileCount = filterActive ? finalGroup.Count(f => AudioFormatHelper.GetAudioCodecFromExtension(f.Extension ?? Path.GetExtension(f.Filename) ?? "") != AudioFormat.Unknown) : finalGroup.Count(); if (fileCount < searchTextData.MinimumFiles) { _logger.Trace($"Filtered (too few): {directoryGroup.Key} ({fileCount}/{searchTextData.MinimumFiles} {(filterActive ? "audio tracks" : "files")})"); continue; } if (searchTextData.MaximumFiles.HasValue && fileCount > searchTextData.MaximumFiles.Value) { _logger.Trace($"Filtered (too many): {directoryGroup.Key} ({fileCount}/{searchTextData.MaximumFiles} {(filterActive ? "audio tracks" : "files")})"); continue; } } AlbumData albumData = _itemsParser.CreateAlbumData(searchResponse.Id, finalGroup, searchTextData, folderData, Settings, searchTextData.MinimumFiles); albumDatas.Add(albumData); } } _sentry.UpdateSearchResultCount(searchResponse.Id, albumDatas.Count); RemoveSearch(searchResponse.Id, albumDatas.Count != 0 && searchTextData.Interactive); } catch (Exception ex) { _logger.Error(ex, "Failed to parse Slskd search response."); } return albumDatas.OrderByDescending(x => x.Priotity).Select(a => a.ToReleaseInfo()).ToList(); } private IGrouping? TryExpandDirectory(SlskdSearchData searchTextData, IGrouping directoryGroup, SlskdFolderData folderData) { if (string.IsNullOrEmpty(searchTextData.Artist) || string.IsNullOrEmpty(searchTextData.Album)) return null; bool artistMatch = Fuzz.PartialRatio(folderData.Artist, searchTextData.Artist) > 85; bool albumMatch = Fuzz.PartialRatio(folderData.Album, searchTextData.Album) > 85; if (!artistMatch || !albumMatch) return null; SlskdFileData? originalTrack = directoryGroup.FirstOrDefault(x => AudioFormatHelper.GetAudioCodecFromExtension(x.Extension?.ToLowerInvariant() ?? Path.GetExtension(x.Filename) ?? "") != AudioFormat.Unknown); if (originalTrack == null) return null; _logger.Trace($"Expanding directory for: {folderData.Username}:{directoryGroup.Key}"); SlskdRequestGenerator? requestGenerator = _indexer.GetExtendedRequestGenerator() as SlskdRequestGenerator; IGrouping? expandedGroup = requestGenerator?.ExpandDirectory(folderData.Username, directoryGroup.Key, originalTrack).GetAwaiter().GetResult(); if (expandedGroup != null) { _logger.Debug($"Successfully expanded directory to {expandedGroup.Count()} files"); return expandedGroup; } else { _logger.Warn($"Failed to expand directory for {folderData.Username}:{directoryGroup.Key}"); } return null; } public void RemoveSearch(string searchId, bool delay = false) { Task.Run(async () => { try { if (delay) { _interactiveResults.TryGetValue(_indexer.Definition.Id, out string? staleId); _interactiveResults[_indexer.Definition.Id] = searchId; if (staleId != null) searchId = staleId; else return; } await ExecuteRemovalAsync(Settings, searchId); } catch (HttpException ex) { _logger.Error(ex, $"Failed to remove slskd search with ID: {searchId}"); } }); } public void Handle(AlbumGrabbedEvent message) { if (!_interactiveResults.TryGetValue(message.Album.Release.IndexerId, out string? selectedId) || !message.Album.Release.InfoUrl.EndsWith(selectedId)) return; ExecuteRemovalAsync((SlskdSettings)_indexerFactory.Value.Get(message.Album.Release.IndexerId).Settings, selectedId).GetAwaiter().GetResult(); _interactiveResults.Remove(message.Album.Release.IndexerId); } public void Handle(ApplicationShutdownRequested message) { foreach (int indexerId in _interactiveResults.Keys.ToList()) { if (_interactiveResults.TryGetValue(indexerId, out string? selectedId)) { ExecuteRemovalAsync((SlskdSettings)_indexerFactory.Value.Get(indexerId).Settings, selectedId).GetAwaiter().GetResult(); _interactiveResults.Remove(indexerId); } } } public static void InvalidIgnoreCache(string path) => _ignoreListCache.Remove(path); private HashSet GetRateLimitedUsers() { if (Settings.MaxGrabsPerUser <= 0 && Settings.MaxQueuedPerUser <= 0) return []; lock (_rateLimitLock) { if (DateTime.UtcNow - _rateLimitCacheTimestamp < TimeSpan.FromSeconds(15)) return _rateLimitedUsersCache; HashSet blocked = new(StringComparer.OrdinalIgnoreCase); if (Settings.MaxGrabsPerUser > 0) foreach ((string? user, int count) in GetGrabCounts()) if (count >= Settings.MaxGrabsPerUser) blocked.Add(user); if (Settings.MaxQueuedPerUser > 0) foreach ((string? user, int count) in GetQueuedCounts()) if (count >= Settings.MaxQueuedPerUser) blocked.Add(user); _rateLimitedUsersCache = blocked; _rateLimitCacheTimestamp = DateTime.UtcNow; return blocked; } } private Dictionary GetGrabCounts() { DateTime since = (GrabLimitIntervalType)Settings.GrabLimitInterval switch { GrabLimitIntervalType.Hour => DateTime.UtcNow.AddHours(-1), GrabLimitIntervalType.Week => DateTime.UtcNow.AddDays(-7), _ => DateTime.UtcNow.Date }; int? indexerId = _indexer.Definition?.Id; Dictionary counts = new(StringComparer.OrdinalIgnoreCase); IEnumerable downloadIds = _historyService.Since(since, EntityHistoryEventType.Grabbed) .Select(h => h.DownloadId) .Where(id => !string.IsNullOrWhiteSpace(id)) .Distinct(StringComparer.OrdinalIgnoreCase); foreach (string downloadId in downloadIds) { DownloadHistory? grab = _downloadHistoryService.GetLatestGrab(downloadId); if (grab == null) continue; if (!string.Equals(grab.Protocol, nameof(SoulseekDownloadProtocol), StringComparison.OrdinalIgnoreCase)) continue; if (indexerId.HasValue && grab.IndexerId != indexerId.Value) continue; string? username = ExtractUsernameFromUrl(grab.Release?.DownloadUrl); if (username != null) counts[username] = counts.GetValueOrDefault(username) + 1; } return counts; } private Dictionary GetQueuedCounts() { string? indexerName = _indexer.Definition?.Name; Dictionary counts = new(StringComparer.OrdinalIgnoreCase); HashSet seen = new(StringComparer.OrdinalIgnoreCase); foreach (Queue item in _queueService.GetQueue()) { if (!string.Equals(item.Protocol, nameof(SoulseekDownloadProtocol), StringComparison.OrdinalIgnoreCase)) continue; if (!string.IsNullOrWhiteSpace(indexerName) && !string.Equals(item.Indexer, indexerName, StringComparison.OrdinalIgnoreCase)) continue; if (string.IsNullOrWhiteSpace(item.DownloadId) || !seen.Add(item.DownloadId)) continue; DownloadHistory? grab = _downloadHistoryService.GetLatestGrab(item.DownloadId); if (grab == null) continue; string? username = ExtractUsernameFromUrl(grab.Release?.DownloadUrl); if (username != null) counts[username] = counts.GetValueOrDefault(username) + 1; } return counts; } private static string? ExtractUsernameFromUrl(string? url) { if (string.IsNullOrWhiteSpace(url)) return null; int lastSlash = url.LastIndexOf('/'); if (lastSlash < 0 || lastSlash >= url.Length - 1) return null; return Uri.UnescapeDataString(url[(lastSlash + 1)..]); } private async Task ExecuteRemovalAsync(SlskdSettings settings, string searchId) { try { HttpRequest request = new HttpRequestBuilder($"{settings.BaseUrl}/api/v0/searches/{searchId}") .SetHeader("X-API-KEY", settings.ApiKey) .Build(); request.Method = HttpMethod.Delete; await _httpClient.ExecuteAsync(request); } catch (HttpException ex) { _logger.Error(ex, $"Failed to remove slskd search with ID: {searchId}"); } } private HashSet? GetIgnoredUsers(string? ignoreListPath) { if (string.IsNullOrWhiteSpace(ignoreListPath) || !File.Exists(ignoreListPath)) return null; try { FileInfo fileInfo = new(ignoreListPath); long fileSize = fileInfo.Length; if (_ignoreListCache.TryGetValue(ignoreListPath, out (HashSet IgnoredUsers, long LastFileSize) cached) && cached.LastFileSize == fileSize) { _logger.Trace($"Using cached ignore list from: {ignoreListPath} with {cached.IgnoredUsers.Count} users"); return cached.IgnoredUsers; } HashSet ignoredUsers = SlskdTextProcessor.ParseListContent(File.ReadAllText(ignoreListPath)); _ignoreListCache[ignoreListPath] = (ignoredUsers, fileSize); _logger.Trace($"Loaded ignore list with {ignoredUsers.Count} users from: {ignoreListPath}"); return ignoredUsers; } catch (Exception ex) { _logger.Warn(ex, $"Failed to load ignore list from: {ignoreListPath}"); return null; } } } } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdItemsParser.cs ================================================ using FuzzySharp; using Newtonsoft.Json; using NLog; using NzbDrone.Core.Indexers; using System.Collections.Concurrent; using System.Text.RegularExpressions; using Tubifarry.Core.Model; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Soulseek { public partial class SlskdItemsParser : ISlskdItemsParser { private readonly Logger _logger; private readonly ISentryHelper _sentry; // Fuzzy matching threshold constants private const int FuzzyArtistPartialThreshold = 90; private const int FuzzyArtistTokenSortThreshold = 85; private const int FuzzyAlbumPartialThreshold = 85; private const int FuzzyAlbumTokenSortThreshold = 80; private const int FuzzyCombinedThreshold = 85; private static readonly ConcurrentDictionary<(string, string), (int Partial, int TokenSort)> FuzzyCache = new(); private const int MaxCacheSize = 10000; private static readonly Dictionary _textNumbers = new(StringComparer.OrdinalIgnoreCase) { { "one", "1" }, { "two", "2" }, { "three", "3" }, { "four", "4" }, { "five", "5" }, { "six", "6" }, { "seven", "7" }, { "eight", "8" }, { "nine", "9" }, { "ten", "10" } }; private static readonly Dictionary _romanNumerals = new() { { 'I', 1 }, { 'V', 5 }, { 'X', 10 }, { 'L', 50 }, { 'C', 100 }, { 'D', 500 }, { 'M', 1000 } }; private static readonly string[] _nonArtistFolders = [ "music", "mp3", "flac", "audio", "compilations", "soundtracks", "pop", "rock", "jazz", "classical", "various", "downloads" ]; public SlskdItemsParser(ISentryHelper sentry, Logger logger) { _sentry = sentry; _logger = logger; } public SlskdFolderData ParseFolderName(string folderPath) { string[] pathComponents = SplitPathIntoComponents(folderPath); (string? artist, string? album, string? year) = ParseFromRegexPatterns(pathComponents); if (string.IsNullOrEmpty(artist) && pathComponents.Length >= 2) artist = GetArtistFromParentFolder(pathComponents); if (string.IsNullOrEmpty(album) && pathComponents.Length > 0) album = CleanComponent(pathComponents[^1]); if (string.IsNullOrEmpty(year)) year = ExtractYearFromPath(folderPath); return new SlskdFolderData( Path: folderPath, Artist: artist ?? "Unknown Artist", Album: album ?? "Unknown Album", Year: year ?? string.Empty, Username: string.Empty, HasFreeUploadSlot: false, UploadSpeed: 0, LockedFileCount: 0, LockedFiles: [], QueueLength: 0, Token: 0, FileCount: 0, Files: []); } public AlbumData CreateAlbumData(string searchId, IGrouping directory, SlskdSearchData searchData, SlskdFolderData folderData, SlskdSettings? settings = null, int expectedTrackCount = 0) { string dirNameNorm = NormalizeString(directory.Key); string searchArtistNorm = NormalizeString(searchData.Artist ?? ""); string searchAlbumNorm = NormalizeString(searchData.Album ?? ""); _logger.Trace($"Creating album data - Dir: '{dirNameNorm}', Search artist: '{searchArtistNorm}', Search album: '{searchAlbumNorm}'"); // Calculate fuzzy scores with caching for performance (int fuzzyArtistPartial, int fuzzyArtistTokenSort) = GetCachedFuzzyScores(dirNameNorm, searchArtistNorm); (int fuzzyAlbumPartial, int fuzzyAlbumTokenSort) = GetCachedFuzzyScores(dirNameNorm, searchAlbumNorm); bool isVolumeSearch = !string.IsNullOrEmpty(searchData.Album) && VolumeRegex().Match(searchData.Album).Success; bool isAlbumMatch = isVolumeSearch ? CheckVolumeSeriesMatch(directory.Key, searchData.Album) : !string.IsNullOrEmpty(searchAlbumNorm) && (fuzzyAlbumPartial > FuzzyAlbumPartialThreshold || fuzzyAlbumTokenSort > FuzzyAlbumTokenSortThreshold); bool isArtistMatch = IsFuzzyArtistMatch(dirNameNorm, searchArtistNorm); if (!isArtistMatch && !isAlbumMatch && !string.IsNullOrEmpty(searchData.Artist) && !string.IsNullOrEmpty(searchData.Album)) { string combinedSearch = NormalizeString($"{searchData.Artist} {searchData.Album}"); (int combinedPartial, _) = GetCachedFuzzyScores(dirNameNorm, combinedSearch); isAlbumMatch = combinedPartial > FuzzyCombinedThreshold; } _logger.Debug($"Match results - Artist: {isArtistMatch}, Album: {isAlbumMatch}"); // Determine final values for artist, album, year string finalArtist = DetermineFinalArtist(isArtistMatch, folderData, searchData); string finalAlbum = DetermineFinalAlbum(isAlbumMatch, folderData, searchData); string finalYear = folderData.Year; (AudioFormat Codec, int? BitRate, int? BitDepth, int? SampleRate, long TotalSize, int TotalDuration) = AnalyzeAudioQuality(directory); string qualityInfo = FormatQualityInfo(Codec, BitRate, BitDepth, SampleRate); List? filesToDownload = directory.GroupBy(f => f.Filename?[..f.Filename.LastIndexOf('\\')]).FirstOrDefault(g => g.Key == directory.Key)?.ToList(); int actualTrackCount = filesToDownload?.Count ?? 0; _logger.Trace($"Audio: {Codec}, BitRate: {BitRate}, BitDepth: {BitDepth}, Files: {actualTrackCount}"); string infoUrl = settings != null ? $"{(string.IsNullOrEmpty(settings.ExternalUrl) ? settings.BaseUrl : settings.ExternalUrl)}/searches/{searchId}" : ""; string? edition = ExtractEdition(folderData.Path)?.ToUpper(); int priority = folderData.CalculatePriority(expectedTrackCount); string regexMatchType = DetermineRegexMatchType(folderData.Path); List? directoryFiles = filesToDownload?.Select(f => f.Filename ?? "").Where(f => !string.IsNullOrEmpty(f)).ToList(); _sentry.LogParseResult( searchId, folderData.Path, regexMatchType, fuzzyArtistPartial, fuzzyAlbumPartial, fuzzyArtistTokenSort, fuzzyAlbumTokenSort, priority, Codec.ToString(), BitRate ?? 0, BitDepth ?? 0, expectedTrackCount, actualTrackCount, folderData.Username, folderData.HasFreeUploadSlot, folderData.QueueLength, directoryFiles, searchData.Interactive); return new AlbumData("Slskd", nameof(SoulseekDownloadProtocol)) { AlbumId = $"/api/v0/transfers/downloads/{folderData.Username}", ArtistName = finalArtist, AlbumName = finalAlbum, ReleaseDate = finalYear, ReleaseDateTime = string.IsNullOrEmpty(finalYear) || !int.TryParse(finalYear, out int yearInt) ? DateTime.MinValue : new DateTime(yearInt, 1, 1), Codec = Codec, BitDepth = BitDepth ?? 0, Bitrate = (Codec == AudioFormat.MP3 ? AudioFormatHelper.RoundToStandardBitrate(BitRate ?? 0) : BitRate) ?? 0, Size = TotalSize, InfoUrl = infoUrl, ExplicitContent = ExtractExplicitTag(folderData.Path), Priotity = priority, CustomString = JsonConvert.SerializeObject(filesToDownload), ExtraInfo = [edition, $"👤 {folderData.Username} ", $"{(folderData.HasFreeUploadSlot ? "⚡" : "❌")} {folderData.UploadSpeed / 1024.0 / 1024.0:F2}MB/s ", folderData.QueueLength == 0 ? "" : $"📋 {folderData.QueueLength}"], Duration = TotalDuration }; } private static string[] SplitPathIntoComponents(string path) => path.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries); private static (string? artist, string? album, string? year) ParseFromRegexPatterns(string[] pathComponents) { if (pathComponents.Length == 0) return (null, null, null); string lastComponent = pathComponents[^1]; // Try artist-album-year pattern Match? match = TryMatchRegex(lastComponent, ArtistAlbumYearRegex()); if (match != null) { return ( match.Groups["artist"].Success ? match.Groups["artist"].Value.Trim() : null, match.Groups["album"].Success ? match.Groups["album"].Value.Trim() : null, match.Groups["year"].Success ? match.Groups["year"].Value.Trim() : null); } // Try year-artist-album pattern match = TryMatchRegex(lastComponent, YearArtistAlbumRegex()); if (match != null) { return ( match.Groups["artist"].Success ? match.Groups["artist"].Value.Trim() : null, match.Groups["album"].Success ? match.Groups["album"].Value.Trim() : null, match.Groups["year"].Success ? match.Groups["year"].Value.Trim() : null); } // Try album-year pattern match = TryMatchRegex(lastComponent, AlbumYearRegex()); if (match?.Groups["album"].Success == true) { string? artist = null; if (pathComponents.Length >= 2) artist = GetArtistFromParentFolder(pathComponents); return (artist, match.Groups["album"].Value.Trim(), match.Groups["year"].Success ? match.Groups["year"].Value.Trim() : null); } return (null, null, null); } private static string? GetArtistFromParentFolder(string[] pathComponents) { if (pathComponents.Length < 2) return null; string parentFolder = pathComponents[^2]; if (!_nonArtistFolders.Contains(parentFolder.ToLowerInvariant())) return parentFolder; return null; } private bool CheckVolumeSeriesMatch(string directoryPath, string? searchAlbum) { if (string.IsNullOrEmpty(searchAlbum)) return false; bool isVolumeSeries = VolumeRegex().Match(searchAlbum).Success; if (!isVolumeSeries) return false; Match? searchMatch = VolumeRegex().Match(searchAlbum); Match dirMatch = VolumeRegex().Match(directoryPath); if (!dirMatch.Success) return false; string? normSearchVol = NormalizeVolume(searchMatch.Value); string? normDirVol = NormalizeVolume(dirMatch.Value); string? searchBaseAlbum = VolumeRegex().Replace(searchAlbum, "").Trim(); string? dirBaseAlbum = VolumeRegex().Replace(directoryPath, "").Trim(); bool baseAlbumMatch = Fuzz.PartialRatio( NormalizeString(dirBaseAlbum), NormalizeString(searchBaseAlbum)) > FuzzyAlbumPartialThreshold; return baseAlbumMatch && normSearchVol.Equals(normDirVol, StringComparison.OrdinalIgnoreCase); } public string NormalizeVolume(string volume) { _logger.Trace($"Normalizing volume: '{volume}'"); if (_textNumbers.TryGetValue(volume, out string? textNum)) return textNum; if (int.TryParse(volume, out int num)) return num.ToString(); string normalizedRoman = NormalizeRomanNumeral(volume); if (RomanNumeralRegex().IsMatch(normalizedRoman)) { int value = ConvertRomanToNumber(normalizedRoman); if (value > 0) return value.ToString(); } Match rangeMatch = VolumeRangeRegex().Match(volume); if (rangeMatch.Success && int.TryParse(rangeMatch.Groups[1].Value, out int firstNum)) return firstNum.ToString(); return volume.Trim().ToUpperInvariant(); } private static string NormalizeRomanNumeral(string roman) { if (string.IsNullOrEmpty(roman)) return string.Empty; return roman.Trim().ToUpperInvariant() .Replace("IIII", "IV") // 4 .Replace("VIIII", "IX") // 9 .Replace("XXXX", "XL") // 40 .Replace("LXXXX", "XC") // 90 .Replace("CCCC", "CD") // 400 .Replace("DCCCC", "CM"); // 900 } private int ConvertRomanToNumber(string roman) { roman = roman.ToUpperInvariant(); int total = 0; int prevValue = 0; for (int i = roman.Length - 1; i >= 0; i--) { if (!_romanNumerals.TryGetValue(roman[i], out int currentValue)) return 0; if (currentValue < prevValue) total -= currentValue; else total += currentValue; prevValue = currentValue; } if (total <= 0 || total > 5000) return 0; _logger.Trace($"Roman numeral '{roman}' converted to: {total}"); return total; } public static string NormalizeString(string input) { if (string.IsNullOrEmpty(input)) return string.Empty; string normalized = NormalizeCharactersRegex().Replace(input, " "); normalized = RemoveNonAlphanumericRegex().Replace(normalized, ""); normalized = ReduceWhitespaceRegex().Replace(normalized.ToLowerInvariant(), " ").Trim(); normalized = RemoveWordsRegex().Replace(normalized, ""); return ReduceWhitespaceRegex().Replace(normalized, " ").Trim(); } private static string CleanComponent(string component) { if (string.IsNullOrEmpty(component)) return string.Empty; component = CleanComponentRegex().Replace(component, ""); return ReduceWhitespaceRegex().Replace(component.Trim(), " "); } private bool ExtractExplicitTag(string path) { Match match = ExplicitTagRegex().Match(path); if (match.Success) { if (match.Groups["negation"].Success && !string.IsNullOrWhiteSpace(match.Groups["negation"].Value)) { _logger.Trace($"Found negated explicit tag in path, skipping: {match.Value}"); return false; } _logger.Trace($"Extracted explicit tag from path: {path}"); return true; } return false; } private static string? ExtractEdition(string path) { Match match = EditionRegex().Match(path); return match.Success ? match.Groups["edition"].Value.Trim() : null; } private static string? ExtractYearFromPath(string path) { Match yearMatch = YearExtractionRegex().Match(path); return yearMatch.Success ? yearMatch.Groups["year"].Value : null; } private static Match? TryMatchRegex(string input, Regex regex) { Match match = regex.Match(input); return match.Success ? match : null; } private static bool IsFuzzyArtistMatch(string dirNameNorm, string searchArtistNorm) { if (string.IsNullOrEmpty(searchArtistNorm)) return false; (int partial, int tokenSort) = GetCachedFuzzyScores(dirNameNorm, searchArtistNorm); return partial > FuzzyArtistPartialThreshold || tokenSort > FuzzyArtistTokenSortThreshold; } private static bool IsFuzzyAlbumMatch(string dirNameNorm, string searchAlbumNorm, bool volumeMatch) { if (string.IsNullOrEmpty(searchAlbumNorm)) return false; if (volumeMatch) return true; (int partial, int tokenSort) = GetCachedFuzzyScores(dirNameNorm, searchAlbumNorm); return partial > FuzzyAlbumPartialThreshold || tokenSort > FuzzyAlbumTokenSortThreshold; } /// /// Gets fuzzy matching scores with caching to avoid redundant calculations. /// private static (int Partial, int TokenSort) GetCachedFuzzyScores(string str1, string str2) { if (string.IsNullOrEmpty(str1) || string.IsNullOrEmpty(str2)) return (0, 0); // Create a consistent cache key (alphabetically ordered to ensure cache hits) (string, string) cacheKey = string.Compare(str1, str2, StringComparison.Ordinal) < 0 ? (str1, str2) : (str2, str1); return FuzzyCache.GetOrAdd(cacheKey, key => { // Clear cache if it gets too large to prevent memory issues if (FuzzyCache.Count > MaxCacheSize) { FuzzyCache.Clear(); } return ( Fuzz.PartialRatio(key.Item1, key.Item2), Fuzz.TokenSortRatio(key.Item1, key.Item2) ); }); } public static string DetermineRegexMatchType(string folderPath) { string[] pathComponents = SplitPathIntoComponents(folderPath); if (pathComponents.Length == 0) return "none"; string lastComponent = pathComponents[^1]; if (TryMatchRegex(lastComponent, ArtistAlbumYearRegex()) != null) return "ArtistAlbumYear"; if (TryMatchRegex(lastComponent, YearArtistAlbumRegex()) != null) return "YearArtistAlbum"; if (TryMatchRegex(lastComponent, AlbumYearRegex()) != null) return "AlbumYear"; return "none"; } private static string DetermineFinalArtist(bool isArtistMatch, SlskdFolderData folderData, SlskdSearchData searchData) { if (isArtistMatch && !string.IsNullOrEmpty(searchData.Artist)) return searchData.Artist; if (!string.IsNullOrEmpty(folderData.Artist)) return folderData.Artist; return searchData.Artist ?? "Unknown Artist"; } private static string DetermineFinalAlbum(bool isAlbumMatch, SlskdFolderData folderData, SlskdSearchData searchData) { if (isAlbumMatch && !string.IsNullOrEmpty(searchData.Album)) { Match folderVersion = VolumeRegex().Match(folderData.Album ?? ""); Match searchVersion = VolumeRegex().Match(searchData.Album); return folderVersion.Success && !searchVersion.Success ? $"{searchData.Album} {folderVersion.Value}" : searchData.Album; } if (!string.IsNullOrEmpty(folderData.Album)) return folderData.Album; return searchData.Album ?? "Unknown Album"; } private (AudioFormat Codec, int? BitRate, int? BitDepth, int? SampleRate, long TotalSize, int TotalDuration) AnalyzeAudioQuality(IGrouping directory) { string? commonExt = GetMostCommonExtension(directory); long totalSize = directory.Sum(f => f.Size); int totalDuration = directory.Sum(f => f.Length ?? 0); int? commonBitRate = directory.GroupBy(f => f.BitRate).OrderByDescending(g => g.Count()).FirstOrDefault()?.Key; int? commonBitDepth = directory.GroupBy(f => f.BitDepth).OrderByDescending(g => g.Count()).FirstOrDefault()?.Key; int? commonSampleRate = directory.GroupBy(f => f.SampleRate).OrderByDescending(g => g.Count()).FirstOrDefault()?.Key; if (!commonBitRate.HasValue && totalDuration > 0) { commonBitRate = (int)(totalSize * 8 / (totalDuration * 1000)); _logger.Trace($"Calculated bitrate: {commonBitRate}"); } AudioFormat codec = AudioFormatHelper.GetAudioCodecFromExtension(commonExt ?? ""); return (codec, commonBitRate, commonBitDepth, commonSampleRate, totalSize, totalDuration); } public static string? GetMostCommonExtension(IEnumerable files) { List extensions = files .Select(f => string.IsNullOrEmpty(f.Extension) ? Path.GetExtension(f.Filename)?.TrimStart('.').ToLowerInvariant() : f.Extension) .Where(ext => !string.IsNullOrEmpty(ext)) .ToList(); if (extensions.Count == 0) return null; return extensions .GroupBy(ext => ext) .OrderByDescending(g => g.Count()) .Select(g => g.Key) .FirstOrDefault(); } private static string FormatQualityInfo(AudioFormat codec, int? bitRate, int? bitDepth, int? sampleRate) { if (codec == AudioFormat.MP3 && bitRate.HasValue) return $"{codec} {bitRate}kbps"; if (bitDepth.HasValue && sampleRate.HasValue) return $"{codec} {bitDepth}bit/{sampleRate / 1000}kHz"; return codec.ToString(); } [GeneratedRegex(@"(?ix) \[(?:FLAC|MP3|320|WEB|CD)[^\]]*\]| # Audio format tags \(\d{5,}\)| # Long numbers in parentheses \(\d+bit[\/\s]\d+[^\)]*\)| # Bit depth/sample rate \((?:DELUXE_)?EDITION\)| # Edition markers \s*\([^)]*edition[^)]*\)| # Any edition in parentheses \((?:Album|Single|EP|LP)\)| # Release type \s*\(remaster(?:ed)?\)| # Remaster tags \s*[\(\[][^)\]]*(?:version|reissue)[^)\]]*[\)\]]| # Version/reissue in brackets/parens \s*\d{4}\s*remaster| # Year remaster \s-\s.*$", RegexOptions.ExplicitCapture | RegexOptions.Compiled, "de-DE")] private static partial Regex CleanComponentRegex(); [GeneratedRegex(@"(?ix) (?<=\b(?:volume|vol|part|pt|chapter|ep|sampler|remix(?:es)?|mix(?:es)?|edition|ed|version|ver|v|release|issue|series|no|num|phase|stage|book|side|disc|cd|dvd|track|season|installment|\#)\s*[.,\-_:#]*\s*) (\d+(?:\.\d+)?|[IVXLCDM]+|\d+(?:[-to&]\d+)?|one|two|three|four|five|six|seven|eight|nine|ten)(?!\w)| (\d+(?:\.\d+)?|[IVXLCDM]+)(?=\s*$)", RegexOptions.ExplicitCapture | RegexOptions.Compiled, "de-DE")] private static partial Regex VolumeRegex(); [GeneratedRegex(@"^(?[^(\[]+)(?:\s*[\(\[](?19\d{2}|20\d{2})[\)\]])?", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex AlbumYearRegex(); [GeneratedRegex(@"^(?19\d{2}|20\d{2})\s*-\s*(?[^-]+)\s*-\s*(?.+)(?:\s*[\(\[].+[\)\]])*$", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex YearArtistAlbumRegex(); [GeneratedRegex(@"^(?[^-]+)\s*-\s*(?[^(\[]+)(?:\s*[\(\[](?19\d{2}|20\d{2})[\)\]])?", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex ArtistAlbumYearRegex(); [GeneratedRegex(@"(?19\d{2}|20\d{2})", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex YearExtractionRegex(); [GeneratedRegex(@"(?:^|\s|\(|\[)(?Non-?|Not\s+)?Explicit(?:\s|\)|\]|$)", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture | RegexOptions.Compiled, "de-DE")] private static partial Regex ExplicitTagRegex(); [GeneratedRegex(@"[\(\[](?(?:(?:Super\s+)?Deluxe|Limited|Special|Expanded|Extended|Anniversary|Remaster(?:ed)?|Live|Acoustic|Unplugged|Japanese|Bonus|Instrumental|Collector'?s|Metal|Platinum|Gold|Clean|Tour|Censored|Uncensored|\d*\s*CD)(?:\s+(?:Edition|Version|Album|Tracks?|Exclusive))?|[^)\]]+?\s+(?:Edition|Version))[\)\]]", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex EditionRegex(); [GeneratedRegex(@"\b(?:the|a|an|feat|featuring|ft|presents|pres|with|and)\b", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture | RegexOptions.Compiled, "de-DE")] private static partial Regex RemoveWordsRegex(); [GeneratedRegex(@"(\d+)(?:[-to&]\s*\d+)?", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex VolumeRangeRegex(); [GeneratedRegex(@"^M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$", RegexOptions.ExplicitCapture | RegexOptions.Compiled)] private static partial Regex RomanNumeralRegex(); [GeneratedRegex(@"\s+", RegexOptions.Compiled)] private static partial Regex ReduceWhitespaceRegex(); [GeneratedRegex(@"[^\w\s$-]", RegexOptions.Compiled)] private static partial Regex RemoveNonAlphanumericRegex(); [GeneratedRegex(@"[._/]+", RegexOptions.Compiled)] private static partial Regex NormalizeCharactersRegex(); } } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdRequestGenerator.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Common.Serializer; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Music; using System.Net; using System.Text.Json; using System.Text.Json.Nodes; using Tubifarry.Core.Replacements; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; using Tubifarry.Indexers.Soulseek.Search.Core; namespace Tubifarry.Indexers.Soulseek { internal class SlskdRequestGenerator : IIndexerRequestGenerator { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true }; private readonly SlskdIndexer _indexer; private readonly Logger _logger; private readonly IHttpClient _client; private readonly ISlskdSearchChain _searchPipeline; private readonly ISentryHelper _sentry; private readonly HashSet _processedSearches = new(StringComparer.OrdinalIgnoreCase); private SlskdSettings Settings => _indexer.Settings; public SlskdRequestGenerator(SlskdIndexer indexer, ISlskdSearchChain searchPipeline, IHttpClient client, ISentryHelper sentry) { _indexer = indexer; _client = client; _searchPipeline = searchPipeline; _sentry = sentry; _logger = NzbDroneLogger.GetLogger(this); } public IndexerPageableRequestChain GetRecentRequests() => new LazyIndexerPageableRequestChain(Settings.MinimumResults); public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { _logger.Trace($"Setting up lazy search for album: {searchCriteria.AlbumQuery} by artist: {searchCriteria.ArtistQuery}"); Album? album = searchCriteria.Albums.FirstOrDefault(); List? albumReleases = album?.AlbumReleases?.Value; AlbumRelease? monitoredRelease = albumReleases?.FirstOrDefault(r => r.Monitored); int trackCount = monitoredRelease?.TrackCount ?? (albumReleases?.Any() == true ? albumReleases.Min(x => x.TrackCount) : 0); List tracks = (monitoredRelease ?? albumReleases?.FirstOrDefault(x => x.Tracks?.Value is { Count: > 0 })) ?.Tracks?.Value?.Where(x => !string.IsNullOrEmpty(x.Title)).Select(x => x.Title).ToList() ?? []; _processedSearches.Clear(); SearchContext context = new( Artist: searchCriteria.ArtistQuery, Album: searchCriteria.ArtistQuery != searchCriteria.AlbumQuery ? searchCriteria.AlbumQuery : null, Year: searchCriteria.AlbumYear.ToString(), PrimaryType: GetPrimaryAlbumType(album?.AlbumType), Interactive: searchCriteria.InteractiveSearch, TrackCount: trackCount, Aliases: searchCriteria.Artist?.Metadata.Value.Aliases ?? [], Tracks: tracks, Settings: Settings, ProcessedSearches: _processedSearches, SearchCriteria: searchCriteria); return _searchPipeline.BuildChain(context, ExecuteSearch); } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) { _logger.Debug($"Setting up lazy search for artist: {searchCriteria.CleanArtistQuery}"); Album? album = searchCriteria.Albums.FirstOrDefault(); List? albumReleases = album?.AlbumReleases?.Value; AlbumRelease? monitoredRelease = albumReleases?.FirstOrDefault(r => r.Monitored); int trackCount = monitoredRelease?.TrackCount ?? (albumReleases?.Any() == true ? albumReleases.Min(x => x.TrackCount) : 0); List tracks = (monitoredRelease ?? albumReleases?.FirstOrDefault(x => x.Tracks?.Value is { Count: > 0 })) ?.Tracks?.Value?.Where(x => !string.IsNullOrEmpty(x.Title)).Select(x => x.Title).ToList() ?? []; _processedSearches.Clear(); SearchContext context = new( Artist: searchCriteria.CleanArtistQuery, Album: null, Year: null, PrimaryType: GetPrimaryAlbumType(album?.AlbumType), Interactive: searchCriteria.InteractiveSearch, TrackCount: trackCount, Aliases: searchCriteria.Artist?.Metadata.Value.Aliases ?? [], Tracks: tracks, Settings: Settings, ProcessedSearches: _processedSearches, SearchCriteria: searchCriteria); return _searchPipeline.BuildChain(context, ExecuteSearch); } private IEnumerable ExecuteSearch(SearchQuery query) { string? searchText = query.SearchText ?? SlskdTextProcessor.BuildSearchText(query.Artist, query.Album); if (string.IsNullOrWhiteSpace(searchText)) return []; ISpan? span = _sentry.StartSpan("slskd.search"); _sentry.SetSpanData(span, "search.query", searchText); _sentry.SetSpanData(span, "search.artist", query.Artist); _sentry.SetSpanData(span, "search.album", query.Album); try { IndexerRequest? request = GetRequestsAsync(query, searchText).GetAwaiter().GetResult(); if (request != null) { _logger.Trace($"Successfully generated request for search: {searchText}"); _sentry.FinishSpan(span, SpanStatus.Ok); return [request]; } else { _logger.Trace($"GetRequestsAsync returned null for search: {searchText}"); } } catch (RequestLimitReachedException) { _sentry.FinishSpan(span, SpanStatus.ResourceExhausted); throw; } catch (Exception ex) { _logger.Error(ex, $"Error executing search: {searchText}"); _sentry.FinishSpan(span, ex); } return []; } private async Task GetRequestsAsync(SearchQuery query, string searchText) { try { _logger.Debug($"Search: {searchText}"); dynamic searchData = CreateSearchData(searchText); string searchId = searchData.Id; dynamic searchRequest = CreateSearchRequest(searchData); await ExecuteSearchAsync(searchRequest, searchId); _sentry.LogSearch(searchId, searchText, query.Artist, query.Album, "SlskdSearch", 0); _sentry.LogSearchSettings( searchId, Settings.TrackCountFilter, Settings.NormalizedSeach, Settings.AppendYear, Settings.HandleVolumeVariations, Settings.UseFallbackSearch, Settings.UseTrackFallback, Settings.MinimumResults, !string.IsNullOrEmpty(Settings.SearchTemplates)); _sentry.LogExpectedTracks(searchId, query.Tracks?.ToList() ?? [], query.TrackCount); dynamic request = CreateResultRequest(searchId, query); return new IndexerRequest(request); } catch (HttpException ex) when (ex.Response.StatusCode == HttpStatusCode.Conflict) { throw new RequestLimitReachedException( "Soulseek client is not connected (temporarily banned or disconnected). Indexer disabled.", TimeSpan.FromMinutes(15)); } catch (HttpException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.Warn($"Search request failed for: {searchText}. Error: {ex.Message}"); return null; } catch (Exception ex) { _logger.Error(ex, $"Error generating search request for: {searchText}"); return null; } } private dynamic CreateSearchData(string searchText) => new { Id = Guid.NewGuid().ToString(), Settings.FileLimit, FilterResponses = true, Settings.MaximumPeerQueueLength, Settings.MinimumPeerUploadSpeed, Settings.MinimumResponseFileCount, Settings.ResponseLimit, SearchText = searchText, SearchTimeout = (int)(Settings.TimeoutInSeconds * 1000), }; private HttpRequest CreateSearchRequest(dynamic searchData) { HttpRequest searchRequest = new HttpRequestBuilder($"{Settings.BaseUrl}/api/v0/searches") .SetHeader("X-API-KEY", Settings.ApiKey) .SetHeader("Content-Type", "application/json") .Post() .Build(); searchRequest.SetContent(JsonSerializer.Serialize(searchData)); return searchRequest; } private async Task ExecuteSearchAsync(HttpRequest searchRequest, string searchId) { await _client.ExecuteAsync(searchRequest); await WaitOnSearchCompletionAsync(searchId, TimeSpan.FromSeconds(Settings.TimeoutInSeconds)); } private HttpRequest CreateResultRequest(string searchId, SearchQuery query) { HttpRequest request = new HttpRequestBuilder($"{Settings.BaseUrl}/api/v0/searches/{searchId}") .AddQueryParam("includeResponses", true) .SetHeader("X-API-KEY", Settings.ApiKey) .Build(); TrackCountFilterType filterType = (TrackCountFilterType)Settings.TrackCountFilter; int minimumFiles = filterType switch { TrackCountFilterType.Exact or TrackCountFilterType.Lower or TrackCountFilterType.Unfitting => Math.Max(Settings.MinimumResponseFileCount, query.TrackCount), _ => Settings.MinimumResponseFileCount }; int? maximumFiles = filterType switch { TrackCountFilterType.Exact => query.TrackCount, TrackCountFilterType.Unfitting => query.TrackCount + Math.Max(2, (int)Math.Ceiling(Math.Log(query.TrackCount) * 1.67)), _ => null }; request.ContentSummary = new { Album = query.Album ?? "", Artist = query.Artist, Interactive = query.Interactive, ExpandDirectory = query.ExpandDirectory, MimimumFiles = minimumFiles, MaximumFiles = maximumFiles }.ToJson(); return request; } private async Task WaitOnSearchCompletionAsync(string searchId, TimeSpan timeout) { DateTime startTime = DateTime.UtcNow.AddSeconds(2); string state = "InProgress"; int totalFilesFound = 0; bool hasTimedOut = false; DateTime timeoutEndTime = DateTime.UtcNow; while (state == "InProgress") { TimeSpan elapsed = DateTime.UtcNow - startTime; if (elapsed > timeout && !hasTimedOut) { hasTimedOut = true; timeoutEndTime = DateTime.UtcNow.AddSeconds(20); } else if (hasTimedOut && timeoutEndTime < DateTime.UtcNow) { break; } JsonNode? searchStatus = await GetSearchResultsAsync(searchId); state = searchStatus?["state"]?.GetValue() ?? "InProgress"; int fileCount = searchStatus?["fileCount"]?.GetValue() ?? 0; if (fileCount > totalFilesFound) totalFilesFound = fileCount; double progress = Math.Clamp(fileCount / (double)Settings.FileLimit, 0.0, 1.0); double delay = hasTimedOut && DateTime.UtcNow < timeoutEndTime ? 1.0 : CalculateQuadraticDelay(progress); await Task.Delay(TimeSpan.FromSeconds(delay)); if (state != "InProgress") break; } } private async Task GetSearchResultsAsync(string searchId) { HttpRequest searchResultsRequest = new HttpRequestBuilder($"{Settings.BaseUrl}/api/v0/searches/{searchId}") .SetHeader("X-API-KEY", Settings.ApiKey).Build(); HttpResponse response = await _client.ExecuteAsync(searchResultsRequest); if (response.StatusCode != HttpStatusCode.OK) { _logger.Warn($"Failed to fetch search results for ID {searchId}. Status: {response.StatusCode}, Content: {response.Content}"); return null; } return JsonSerializer.Deserialize(response.Content); } private static double CalculateQuadraticDelay(double progress) { const double a = 16; const double b = -16; const double c = 5; double delay = (a * Math.Pow(progress, 2)) + (b * progress) + c; return Math.Clamp(delay, 0.5, 5); } private static PrimaryAlbumType GetPrimaryAlbumType(string? albumType) { if (string.IsNullOrWhiteSpace(albumType)) return PrimaryAlbumType.Album; PrimaryAlbumType? matchedType = PrimaryAlbumType.All .FirstOrDefault(t => t.Name.Equals(albumType, StringComparison.OrdinalIgnoreCase)); return matchedType ?? PrimaryAlbumType.Album; } public async Task?> ExpandDirectory(string username, string directoryPath, SlskdFileData originalTrack) { try { HttpRequest request = new HttpRequestBuilder($"{Settings.BaseUrl}/api/v0/users/{Uri.EscapeDataString(username)}/directory") .SetHeader("X-API-KEY", Settings.ApiKey) .SetHeader("Content-Type", "application/json") .Post() .Build(); request.SetContent(JsonSerializer.Serialize(new { directory = directoryPath })); HttpResponse response = await _client.ExecuteAsync(request); if (response.StatusCode == HttpStatusCode.OK) { SlskdDirectoryApiResponse[]? directoryResponse = JsonSerializer.Deserialize(response.Content, _jsonOptions); if (directoryResponse?.Length > 0 && directoryResponse[0].Files?.Any() == true) { string originalExtension = originalTrack.Extension?.ToLowerInvariant() ?? ""; List directoryFiles = directoryResponse[0].Files .Where(f => AudioFormatHelper.GetAudioCodecFromExtension(Path.GetExtension(f.Filename)) != AudioFormat.Unknown) .Select(f => { string fileExtension = Path.GetExtension(f.Filename)?.TrimStart('.').ToLowerInvariant() ?? ""; bool sameExtension = fileExtension == originalExtension; return new SlskdFileData( Filename: $"{directoryPath}\\{f.Filename}", BitRate: sameExtension ? originalTrack.BitRate : null, BitDepth: sameExtension ? originalTrack.BitDepth : null, Size: f.Size, Length: sameExtension ? originalTrack.Length : null, Extension: fileExtension, SampleRate: sameExtension ? originalTrack.SampleRate : null, Code: f.Code, IsLocked: false ); }).ToList(); if (directoryFiles.Count != 0) return directoryFiles.GroupBy(f => SlskdTextProcessor.GetDirectoryFromFilename(f.Filename)).First(); } } else { _logger.Debug($"Directory API returned {response.StatusCode} for {username}:{directoryPath}"); } } catch (Exception ex) { _logger.Error(ex, $"Error expanding directory {username}:{directoryPath}"); } return null; } } } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; using Tubifarry.Indexers.Soulseek.Search.Templates; namespace Tubifarry.Indexers.Soulseek { internal class SlskdSettingsValidator : AbstractValidator { public SlskdSettingsValidator() { // Base URL validation RuleFor(c => c.BaseUrl) .ValidRootUrl() .Must(url => !url.EndsWith('/')) .WithMessage("Base URL must not end with a slash ('/')."); // External URL validation (only if not empty) RuleFor(c => c.ExternalUrl) .Must(url => string.IsNullOrEmpty(url) || (Uri.IsWellFormedUriString(url, UriKind.Absolute) && !url.EndsWith('/'))) .WithMessage("External URL must be a valid URL and must not end with a slash ('/')."); // API Key validation RuleFor(c => c.ApiKey) .NotEmpty() .WithMessage("API Key is required."); // File Limit validation RuleFor(c => c.FileLimit) .GreaterThanOrEqualTo(1) .WithMessage("File Limit must be at least 1."); // Maximum Peer Queue Length validation RuleFor(c => c.MaximumPeerQueueLength) .GreaterThanOrEqualTo(100) .WithMessage("Maximum Peer Queue Length must be at least 100."); // Minimum Peer Upload Speed validation RuleFor(c => c.MinimumPeerUploadSpeed) .GreaterThanOrEqualTo(0) .WithMessage("Minimum Peer Upload Speed must be a non-negative value."); // Minimum Response File Count validation RuleFor(c => c.MinimumResponseFileCount) .GreaterThanOrEqualTo(1) .WithMessage("Minimum Response File Count must be at least 1."); // Response Limit validation RuleFor(c => c.ResponseLimit) .GreaterThanOrEqualTo(1) .WithMessage("Response Limit must be at least 1."); // Timeout validation RuleFor(c => c.TimeoutInSeconds) .GreaterThanOrEqualTo(2.0) .WithMessage("Timeout must be at least 2 seconds."); // TrackFallback validation RuleFor(c => c.UseTrackFallback) .Equal(false) .When(c => !c.UseFallbackSearch) .WithMessage("Track Fallback cannot be enabled without Fallback Search."); // Results validation RuleFor(c => c.MinimumResults) .GreaterThanOrEqualTo(0) .WithMessage("Minimum Results must be at least 0."); // Include File Extensions validation RuleFor(c => c.IncludeFileExtensions) .Must(extensions => extensions?.All(ext => !ext.Contains('.')) != false) .WithMessage("File extensions must not contain a dot ('.')."); // Search Templates validation RuleFor(c => c.SearchTemplates) .Must(t => string.IsNullOrWhiteSpace(t) || TemplateEngine.ValidateTemplates(t).Count == 0) .WithMessage((_, t) => string.Join("; ", TemplateEngine.ValidateTemplates(t))); // Ignore List File Path validation RuleFor(c => c.IgnoreListPath) .IsValidPath() .When(c => !string.IsNullOrWhiteSpace(c.IgnoreListPath)) .WithMessage("File path must be valid."); RuleFor(c => c.MaxGrabsPerUser) .GreaterThanOrEqualTo(0) .WithMessage("Max grabs per user must be 0 or greater."); RuleFor(c => c.MaxQueuedPerUser) .GreaterThanOrEqualTo(0) .WithMessage("Max queued per user must be 0 or greater."); } } public class SlskdSettings : IIndexerSettings { private static readonly SlskdSettingsValidator Validator = new(); [FieldDefinition(0, Label = "URL", Type = FieldType.Url, Placeholder = "http://localhost:5030", HelpText = "Slskd instance URL")] public string BaseUrl { get; set; } = "http://localhost:5030"; [FieldDefinition(1, Label = "External URL", Type = FieldType.Url, Placeholder = "https://slskd.example.com", HelpText = "Public URL for interactive search links", Advanced = true)] public string? ExternalUrl { get; set; } = string.Empty; [FieldDefinition(2, Label = "API Key", Type = FieldType.Textbox, Privacy = PrivacyLevel.ApiKey, HelpText = "The API key for your Slskd instance. You can find or set this in the Slskd's settings under 'Options'.", Placeholder = "Enter your API key")] public string ApiKey { get; set; } = string.Empty; [FieldDefinition(3, Type = FieldType.Checkbox, Label = "Audio Files Only", HelpText = "Return only audio file types")] public bool OnlyAudioFiles { get; set; } = true; [FieldDefinition(4, Type = FieldType.Tag, Label = "File Extensions", HelpText = "Additional extensions when Audio Files Only is enabled (without dots)", Advanced = true)] public IEnumerable IncludeFileExtensions { get; set; } = []; [FieldDefinition(6, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Days before release to allow downloads", Advanced = true)] public int? EarlyReleaseLimit { get; set; } = null; [FieldDefinition(7, Type = FieldType.Number, Label = "File Limit", HelpText = "Max files per search", Advanced = true)] public int FileLimit { get; set; } = 10000; [FieldDefinition(8, Type = FieldType.Number, Label = "Max Peer Queue", HelpText = "Max queued requests per peer", Advanced = true)] public int MaximumPeerQueueLength { get; set; } = 1000000; private int _minimumPeerUploadSpeedBytes; [FieldDefinition(9, Type = FieldType.Number, Label = "Min Peer Speed", Unit = "KB/s", HelpText = "Minimum peer upload speed", Advanced = true)] public int MinimumPeerUploadSpeed { get => _minimumPeerUploadSpeedBytes / 1024; set => _minimumPeerUploadSpeedBytes = value * 1024; } [FieldDefinition(10, Type = FieldType.Number, Label = "Min File Count", HelpText = "Minimum files per response", Advanced = true)] public int MinimumResponseFileCount { get; set; } = 1; [FieldDefinition(11, Type = FieldType.Select, SelectOptions = typeof(TrackCountFilterType), Label = "Track Count Filter", HelpText = "Filter releases by track count matching", Advanced = true)] public int TrackCountFilter { get; set; } = (int)TrackCountFilterType.Disabled; [FieldDefinition(12, Type = FieldType.Number, Label = "Response Limit", HelpText = "Max search responses", Advanced = true)] public int ResponseLimit { get; set; } = 100; [FieldDefinition(13, Type = FieldType.Number, Label = "Timeout", Unit = "seconds", HelpText = "Search timeout", Advanced = true)] public double TimeoutInSeconds { get; set; } = 5; [FieldDefinition(14, Type = FieldType.Checkbox, Label = "Append Year", HelpText = "Append the release year to the first search (ignored when templates are set)", Advanced = true)] public bool AppendYear { get; set; } [FieldDefinition(15, Type = FieldType.Checkbox, Label = "Normalize Search", HelpText = "Remove accents and special characters (é→e, ü→u)", Advanced = true)] public bool NormalizedSeach { get; set; } [FieldDefinition(16, Type = FieldType.Checkbox, Label = "Volume Variations", HelpText = "Try alternate volume formats (Vol.1 <-> Volume I)", Advanced = true)] public bool HandleVolumeVariations { get; set; } [FieldDefinition(17, Label = "Enable Fallback Search", Type = FieldType.Checkbox, HelpText = "If no results are found, perform a secondary search using additional metadata.", Advanced = true)] public bool UseFallbackSearch { get; set; } [FieldDefinition(18, Label = "Track Fallback", Type = FieldType.Checkbox, HelpText = "Search by track names as last resort (requires Fallback Search)", Advanced = true)] public bool UseTrackFallback { get; set; } [FieldDefinition(19, Type = FieldType.Number, Label = "Minimum Results", HelpText = "Keep searching until this many results found", Advanced = true)] public int MinimumResults { get; set; } [FieldDefinition(20, Type = FieldType.FilePath, Label = "Ignore List", HelpText = "File with usernames to ignore (one per line)", Advanced = true)] public string? IgnoreListPath { get; set; } = string.Empty; [FieldDefinition(21, Type = FieldType.Textbox, Label = "Search Templates", HelpText = "Custom search pattern (empty=disabled). Use {{Property}} syntax. Valid: AlbumTitle, AlbumYear, Disambiguation, AlbumQuery, CleanAlbumQuery, Artist.*", Advanced = true)] public string? SearchTemplates { get; set; } = string.Empty; [FieldDefinition(22, Type = FieldType.Number, Label = "Grabs per User", HelpText = "Max albums grabbed from one user within the interval. 0 = disabled.", Advanced = true)] public int MaxGrabsPerUser { get; set; } [FieldDefinition(23, Type = FieldType.Select, SelectOptions = typeof(GrabLimitIntervalType), Label = "Grab Limit Interval", HelpText = "Time window for the grab limit.", Advanced = true)] public int GrabLimitInterval { get; set; } = (int)GrabLimitIntervalType.Day; [FieldDefinition(24, Type = FieldType.Number, Label = "Max Queued/User", HelpText = "Max currently queued albums per user. 0 = disabled.", Advanced = true)] public int MaxQueuedPerUser { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum GrabLimitIntervalType { [FieldOption(Label = "Per Hour", Hint = "Rolling 1-hour window")] Hour = 1, [FieldOption(Label = "Per Day", Hint = "Resets at UTC midnight")] Day = 24, [FieldOption(Label = "Per Week", Hint = "Rolling 7-day window")] Week = 168 } public enum TrackCountFilterType { [FieldOption(Label = "Disabled", Hint = "No track count filtering.")] Disabled = 0, [FieldOption(Label = "Exact", Hint = "Only allow releases matching the exact track count.")] Exact = 1, [FieldOption(Label = "Lower", Hint = "Filter out releases with fewer tracks than expected.")] Lower = 2, [FieldOption(Label = "Unfitting", Hint = "Exclude releases with significantly wrong track count.")] Unfitting = 3 } } ================================================ FILE: Tubifarry/Indexers/Soulseek/SlskdTextProcessor.cs ================================================ using System.Globalization; using System.Text; using System.Text.RegularExpressions; namespace Tubifarry.Indexers.Soulseek { /// /// Handles text processing, normalization, and variations for search queries /// public static partial class SlskdTextProcessor { private static readonly Dictionary RomanNumerals = new(StringComparer.OrdinalIgnoreCase) { { "I", 1 }, { "II", 2 }, { "III", 3 }, { "IV", 4 }, { "V", 5 }, { "VI", 6 }, { "VII", 7 }, { "VIII", 8 }, { "IX", 9 }, { "X", 10 }, { "XI", 11 }, { "XII", 12 }, { "XIII", 13 }, { "XIV", 14 }, { "XV", 15 }, { "XVI", 16 }, { "XVII", 17 }, { "XVIII", 18 }, { "XIX", 19 }, { "XX", 20 } }; private static readonly string[] VolumeFormats = { "Volume", "Vol.", "Vol", "v", "V" }; private static readonly Regex PunctuationPattern = new(@"[^\w\s-&]", RegexOptions.Compiled); private static readonly Regex VolumePattern = new(@"(Vol(?:ume)?\.?)\s*([0-9]+|[IVXLCDM]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex RomanNumeralPattern = new(@"\b([IVXLCDM]+)\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); public static string BuildSearchText(string? artist, string? album) => string.Join(" ", new[] { album, artist }.Where(term => !string.IsNullOrWhiteSpace(term)).Select(term => term?.Trim())); public static bool ShouldNormalizeCharacters(string? artist, string? album) { string? normalizedArtist = artist != null ? NormalizeSpecialCharacters(artist) : null; string? normalizedAlbum = album != null ? NormalizeSpecialCharacters(album) : null; return (normalizedArtist != null && !string.Equals(normalizedArtist, artist, StringComparison.OrdinalIgnoreCase)) || (normalizedAlbum != null && !string.Equals(normalizedAlbum, album, StringComparison.OrdinalIgnoreCase)); } public static bool ShouldStripPunctuation(string? artist, string? album) { string? strippedArtist = artist != null ? StripPunctuation(artist) : null; string? strippedAlbum = album != null ? StripPunctuation(album) : null; return (strippedArtist != null && !string.Equals(strippedArtist, artist, StringComparison.OrdinalIgnoreCase)) || (strippedAlbum != null && !string.Equals(strippedAlbum, album, StringComparison.OrdinalIgnoreCase)); } public static bool IsVariousArtists(string artist) => artist.Equals("Various Artists", StringComparison.OrdinalIgnoreCase) || artist.Equals("VA", StringComparison.OrdinalIgnoreCase); public static bool ContainsVolumeReference(string album) => album.Contains("Volume", StringComparison.OrdinalIgnoreCase) || album.Contains("Vol", StringComparison.OrdinalIgnoreCase); public static bool ShouldGenerateRomanVariations(string album) { Match romanMatch = RomanNumeralPattern.Match(album); if (!romanMatch.Success) return false; Match volumeMatch = VolumePattern.Match(album); return !(volumeMatch.Success && volumeMatch.Groups[2].Value.Equals(romanMatch.Groups[1].Value, StringComparison.OrdinalIgnoreCase)); } public static string StripPunctuation(string? input) { if (string.IsNullOrEmpty(input)) return string.Empty; string stripped = PunctuationPattern.Replace(input, ""); return StripPunctuationRegex().Replace(stripped, " ").Trim(); } public static string NormalizeSpecialCharacters(string? input) { if (string.IsNullOrEmpty(input)) return string.Empty; string decomposed = input.Normalize(NormalizationForm.FormD); StringBuilder sb = new(decomposed.Length); foreach (char c in decomposed) { UnicodeCategory cat = CharUnicodeInfo.GetUnicodeCategory(c); if (cat != UnicodeCategory.NonSpacingMark && cat != UnicodeCategory.SpacingCombiningMark && cat != UnicodeCategory.EnclosingMark) sb.Append(c); } return sb.ToString().Normalize(NormalizationForm.FormC); } public static IEnumerable GenerateVolumeVariations(string album) { if (string.IsNullOrEmpty(album)) yield break; Match volumeMatch = VolumePattern.Match(album); if (!volumeMatch.Success) yield break; string volumeFormat = volumeMatch.Groups[1].Value; string volumeNumber = volumeMatch.Groups[2].Value; if (RomanNumerals.TryGetValue(volumeNumber, out int arabicNumber)) { yield return album.Replace(volumeMatch.Value, $"{volumeFormat} {arabicNumber}"); } else if (int.TryParse(volumeNumber, out arabicNumber) && arabicNumber > 0 && arabicNumber <= 20) { KeyValuePair romanPair = RomanNumerals.FirstOrDefault(x => x.Value == arabicNumber); if (!string.IsNullOrEmpty(romanPair.Key)) yield return album.Replace(volumeMatch.Value, $"{volumeFormat} {romanPair.Key}"); } foreach (string format in VolumeFormats) { if (!format.Equals(volumeFormat, StringComparison.OrdinalIgnoreCase)) yield return album.Replace(volumeMatch.Value, $"{format} {volumeNumber}"); } if (album.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length > 3) { string withoutVolume = album.Replace(volumeMatch.Value, "").Trim(); if (withoutVolume.Length > 10) yield return withoutVolume; } } public static IEnumerable GenerateRomanNumeralVariations(string album) { if (string.IsNullOrEmpty(album)) yield break; Match romanMatch = RomanNumeralPattern.Match(album); if (!romanMatch.Success) yield break; Match volumeMatch = VolumePattern.Match(album); if (volumeMatch.Success && volumeMatch.Groups[2].Value.Equals(romanMatch.Groups[1].Value, StringComparison.OrdinalIgnoreCase)) yield break; string romanNumeral = romanMatch.Groups[1].Value; if (RomanNumerals.TryGetValue(romanNumeral, out int arabicNumber)) yield return album.Replace(romanMatch.Value, arabicNumber.ToString()); } public static string GetDirectoryFromFilename(string? filename) { if (string.IsNullOrEmpty(filename)) return ""; int lastBackslashIndex = filename.LastIndexOf('\\'); return lastBackslashIndex >= 0 ? filename[..lastBackslashIndex] : ""; } public static HashSet ParseListContent(string content) { if (string.IsNullOrWhiteSpace(content)) return new HashSet(StringComparer.OrdinalIgnoreCase); return content .Split(['\t', '\r', '\n'], StringSplitOptions.RemoveEmptyEntries) .Where(username => !string.IsNullOrWhiteSpace(username)) .Select(username => username.Trim()) .ToHashSet(StringComparer.OrdinalIgnoreCase); } [GeneratedRegex(@"\s+")] private static partial Regex StripPunctuationRegex(); } } ================================================ FILE: Tubifarry/Indexers/Spotify/SpotifyIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Validation; using Tubifarry.Indexers.YouTube; namespace Tubifarry.Indexers.Spotify { public class SpotifyIndexerSettingsValidator : AbstractValidator { public SpotifyIndexerSettingsValidator() { // Include YouTube validation rules for YouTube Music authentication Include(new YouTubeIndexerSettingsValidator()); // Spotify-specific validation rules RuleFor(x => x.MaxSearchResults) .GreaterThan(0) .LessThanOrEqualTo(50) .WithMessage("Max search results must be between 1 and 50"); RuleFor(x => x.MaxEnrichmentAttempts) .GreaterThan(0) .LessThanOrEqualTo(20) .WithMessage("Max enrichment attempts must be between 1 and 20"); RuleFor(x => x.TrackCountTolerance) .GreaterThanOrEqualTo(0) .LessThanOrEqualTo(50) .WithMessage("Track count tolerance must be between 0 and 50"); RuleFor(x => x.YearTolerance) .GreaterThanOrEqualTo(0) .LessThanOrEqualTo(50) .WithMessage("Year tolerance must be between 0 and 50"); RuleFor(x => x.CustomSpotifyClientId) .NotEmpty() .When(x => !string.IsNullOrWhiteSpace(x.CustomSpotifyClientSecret)) .WithMessage("Custom Spotify Client ID must be provided when Custom Client Secret is set"); RuleFor(x => x.CustomSpotifyClientSecret) .NotEmpty() .When(x => !string.IsNullOrWhiteSpace(x.CustomSpotifyClientId)) .WithMessage("Custom Spotify Client Secret must be provided when Custom Client ID is set"); } } public class SpotifyIndexerSettings : YouTubeIndexerSettings { private static readonly SpotifyIndexerSettingsValidator Validator = new(); [FieldDefinition(10, Label = "Max Search Results", Type = FieldType.Number, HelpText = "Maximum number of results to fetch from Spotify for each search.", Advanced = true)] public int MaxSearchResults { get; set; } = 20; [FieldDefinition(11, Label = "Max Enrichment Attempts", Type = FieldType.Number, HelpText = "Maximum number of YouTube Music albums to check for each Spotify album.", Advanced = true)] public int MaxEnrichmentAttempts { get; set; } = 7; [FieldDefinition(12, Label = "Enable Fuzzy Matching", Type = FieldType.Checkbox, HelpText = "This can help match albums with slight spelling differences but may occasionally match incorrect albums.", Advanced = true)] public bool EnableFuzzyMatching { get; set; } = true; [FieldDefinition(13, Label = "Track Count Tolerance", Type = FieldType.Number, HelpText = "Percentage tolerance for track count differences between Spotify and YouTube Music.", Advanced = true)] public int TrackCountTolerance { get; set; } = 20; [FieldDefinition(14, Label = "Year Tolerance", Type = FieldType.Number, HelpText = "Number of years tolerance for release date differences between Spotify and YouTube Music.", Advanced = true)] public int YearTolerance { get; set; } = 2; [FieldDefinition(15, Label = "Spotify Client ID", Type = FieldType.Textbox, HelpText = "This allows you to use your own Spotify API rate limit quota instead of the shared one. Get your credentials from https://developer.spotify.com/dashboard", Advanced = true)] public string CustomSpotifyClientId { get; set; } = string.Empty; [FieldDefinition(16, Label = "Spotify Client Secret", Type = FieldType.Password, HelpText = "Client ID and Secret must be provided together to use your own credentials.", Advanced = true)] public string CustomSpotifyClientSecret { get; set; } = string.Empty; public override NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Indexers/Spotify/SpotifyParser.cs ================================================ using NLog; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Text.Json; using Tubifarry.Core.Model; namespace Tubifarry.Indexers.Spotify { public interface ISpotifyParser : IParseIndexerResponse { void UpdateSettings(SpotifyIndexerSettings settings); } /// /// Parses Spotify API responses and converts them to album data. /// internal class SpotifyParser(Logger logger, ISpotifyToYouTubeEnricher enricher) : ISpotifyParser { private readonly Logger _logger = logger; private readonly ISpotifyToYouTubeEnricher _enricher = enricher; private SpotifyIndexerSettings? _currentSettings; public void UpdateSettings(SpotifyIndexerSettings settings) { _currentSettings = settings; _enricher.UpdateSettings(settings); } public IList ParseResponse(IndexerResponse indexerResponse) { List releases = []; try { if (string.IsNullOrEmpty(indexerResponse.Content)) { _logger.Warn("Received empty response content from Spotify API"); return releases; } using JsonDocument jsonDoc = JsonDocument.Parse(indexerResponse.Content); JsonElement root = jsonDoc.RootElement; List albums = []; if (root.TryGetProperty("albums", out JsonElement albumsElement)) { if (albumsElement.TryGetProperty("items", out JsonElement itemsElement)) albums.AddRange(ParseAlbumItems(itemsElement)); } else if (root.ValueKind == JsonValueKind.Array) { albums.AddRange(ParseAlbumItems(root)); } foreach (AlbumData album in _enricher.EnrichWithYouTubeData(albums.Take(_currentSettings?.MaxSearchResults ?? 20).ToList()).Where(x => x.Bitrate > 0)) releases.Add(album.ToReleaseInfo()); _logger.Debug($"Successfully converted {releases.Count} albums to releases"); return [.. releases.DistinctBy(x => x.DownloadUrl).OrderByDescending(o => o.PublishDate)]; } catch (Exception ex) { _logger.Error(ex, $"Error parsing Spotify response. Response length: {indexerResponse.Content?.Length ?? 0}"); return releases; } } private List ParseAlbumItems(JsonElement itemsElement) { List albums = []; foreach (JsonElement albumElement in itemsElement.EnumerateArray()) { try { AlbumData albumData = ExtractAlbumInfo(albumElement); albumData.ParseReleaseDate(); albums.Add(albumData); } catch (Exception ex) { _logger.Warn(ex, $"Failed to parse album from Spotify response: {albumElement}"); } } return albums; } private static AlbumData ExtractAlbumInfo(JsonElement album) => new("Spotify", nameof(YoutubeDownloadProtocol)) { AlbumId = album.TryGetProperty("id", out JsonElement idProp) ? idProp.GetString() ?? "UnknownAlbumId" : "UnknownAlbumId", AlbumName = album.TryGetProperty("name", out JsonElement nameProp) ? nameProp.GetString() ?? "UnknownAlbum" : "UnknownAlbum", ArtistName = ExtractArtistName(album), InfoUrl = album.TryGetProperty("external_urls", out JsonElement externalUrlsProp) && externalUrlsProp.TryGetProperty("spotify", out JsonElement spotifyUrlProp) ? spotifyUrlProp.GetString() ?? string.Empty : string.Empty, ReleaseDate = album.TryGetProperty("release_date", out JsonElement releaseDateProp) ? releaseDateProp.GetString() ?? "0000-01-01" : "0000-01-01", ReleaseDatePrecision = album.TryGetProperty("release_date_precision", out JsonElement precisionProp) ? precisionProp.GetString() ?? "day" : "day", TotalTracks = album.TryGetProperty("total_tracks", out JsonElement totalTracksProp) ? totalTracksProp.GetInt32() : 0, ExplicitContent = album.TryGetProperty("explicit", out JsonElement explicitProp) && explicitProp.GetBoolean(), CustomString = ExtractAlbumArtUrl(album), CoverResolution = ExtractCoverResolution(album) }; private static string ExtractArtistName(JsonElement album) { if (!album.TryGetProperty("artists", out JsonElement artistsProp) || artistsProp.GetArrayLength() == 0) return "UnknownArtist"; return artistsProp[0].TryGetProperty("name", out JsonElement nameProp) ? nameProp.GetString() ?? "UnknownArtist" : "UnknownArtist"; } private static string ExtractAlbumArtUrl(JsonElement album) { if (!album.TryGetProperty("images", out JsonElement imagesProp) || imagesProp.GetArrayLength() == 0) return string.Empty; return imagesProp[0].TryGetProperty("url", out JsonElement urlProp) ? urlProp.GetString() ?? string.Empty : string.Empty; } private static string ExtractCoverResolution(JsonElement album) { if (!album.TryGetProperty("images", out JsonElement imagesProp) || imagesProp.GetArrayLength() == 0) return "Unknown"; JsonElement firstImage = imagesProp[0]; bool hasWidth = firstImage.TryGetProperty("width", out JsonElement widthProp); bool hasHeight = firstImage.TryGetProperty("height", out JsonElement heightProp); if (hasWidth && hasHeight) return $"{widthProp.GetInt32()}x{heightProp.GetInt32()}"; return "Unknown"; } } } ================================================ FILE: Tubifarry/Indexers/Spotify/SpotifyRequestGenerator.cs ================================================ using DownloadAssistant.Base; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using Requests; using System.Text.Json; using Tubifarry.Core.Replacements; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.Spotify { public interface ISpotifyRequestGenerator : IIndexerRequestGenerator { void StartTokenRequest(); bool TokenIsExpired(); bool RequestNewToken(); void UpdateSettings(SpotifyIndexerSettings settings); } internal class SpotifyRequestGenerator(Logger logger) : ISpotifyRequestGenerator { private const int MaxPages = 3; private const int DefaultPageSize = 20; private const int DefaultNewReleaseLimit = 30; private string _token = string.Empty; private DateTime _tokenExpiry = DateTime.MinValue; private OwnRequest? _tokenRequest; private SpotifyIndexerSettings? _settings; private readonly Logger _logger = logger; public void UpdateSettings(SpotifyIndexerSettings settings) => _settings = settings; private int PageSize => Math.Min(_settings?.MaxSearchResults ?? DefaultPageSize, 50); private int NewReleaseLimit => Math.Min(_settings?.MaxSearchResults ?? DefaultNewReleaseLimit, 50); public IndexerPageableRequestChain GetRecentRequests() { LazyIndexerPageableRequestChain chain = new(10); try { chain.AddFactory(() => GetRecentReleaseRequests()); } catch (Exception ex) { _logger.Error(ex, "Failed to generate recent release requests"); } return chain; } private IEnumerable GetRecentReleaseRequests() { HandleToken(); string url = $"https://api.spotify.com/v1/browse/new-releases?limit={NewReleaseLimit}"; IndexerRequest req = new(url, HttpAccept.Json); req.HttpRequest.Headers.Add("Authorization", $"Bearer {_token}"); _logger.Trace($"Created request for recent releases: {url}"); yield return req; } public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { _logger.Debug($"Generating search requests for album: '{searchCriteria.AlbumQuery}' by artist: '{searchCriteria.ArtistQuery}'"); LazyIndexerPageableRequestChain chain = new(3); try { // Primary search: album + artist if (!string.IsNullOrEmpty(searchCriteria.AlbumQuery) && !string.IsNullOrEmpty(searchCriteria.ArtistQuery)) { string primaryQuery = $"album:{searchCriteria.AlbumQuery} artist:{searchCriteria.ArtistQuery}"; chain.AddFactory(() => GetAllPagesForQuery(primaryQuery, "album"), 10); } // Fallback search: album only if (!string.IsNullOrEmpty(searchCriteria.AlbumQuery)) { string albumQuery = $"album:{searchCriteria.AlbumQuery}"; chain.AddTierFactory(() => GetAllPagesForQuery(albumQuery, "album"), 5); } // Last resort: artist only (albums by that artist) if (!string.IsNullOrEmpty(searchCriteria.ArtistQuery)) { string artistQuery = $"artist:{searchCriteria.ArtistQuery}"; chain.AddTierFactory(() => GetAllPagesForQuery(artistQuery, "album"), 3); } } catch (Exception ex) { _logger.Error(ex, "Failed to generate album search requests"); } return chain; } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) { _logger.Debug($"Generating search requests for artist: '{searchCriteria.ArtistQuery}'"); LazyIndexerPageableRequestChain chain = new(3); try { if (!string.IsNullOrEmpty(searchCriteria.ArtistQuery)) { string artistQuery = $"artist:{searchCriteria.ArtistQuery}"; chain.AddFactory(() => GetAllPagesForQuery(artistQuery, "album")); } } catch (Exception ex) { _logger.Error(ex, "Failed to generate artist search requests"); } return chain; } private IEnumerable GetAllPagesForQuery(string searchQuery, string searchType) { for (int page = 0; page < MaxPages; page++) { foreach (IndexerRequest request in GetRequests(searchQuery, searchType, page * PageSize)) yield return request; } } private IEnumerable GetRequests(string searchQuery, string searchType, int offset = 0) { HandleToken(); string formattedQuery = Uri.EscapeDataString(searchQuery).Replace(":", "%3A"); string url = $"https://api.spotify.com/v1/search?q={formattedQuery}&type={searchType}&limit={PageSize}&offset={offset}"; IndexerRequest req = new(url, HttpAccept.Json); req.HttpRequest.Headers.Add("Authorization", $"Bearer {_token}"); _logger.Trace($"Created search request for query '{searchQuery}' (offset {offset}): {url}"); yield return req; } private void HandleToken() { if (RequestNewToken()) StartTokenRequest(); if (TokenIsExpired()) _tokenRequest?.Task.GetAwaiter().GetResult(); } public bool TokenIsExpired() => DateTime.Now >= _tokenExpiry; public bool RequestNewToken() => DateTime.Now >= _tokenExpiry.AddMinutes(10); public void StartTokenRequest() { string clientId = !string.IsNullOrWhiteSpace(_settings?.CustomSpotifyClientId) ? _settings.CustomSpotifyClientId : PluginKeys.SpotifyClientId; string clientSecret = !string.IsNullOrWhiteSpace(_settings?.CustomSpotifyClientSecret) ? _settings.CustomSpotifyClientSecret : PluginKeys.SpotifyClientSecret; if (string.IsNullOrEmpty(clientId) || string.IsNullOrEmpty(clientSecret)) { _logger.Warn("Spotify Client ID or Secret is not configured. Cannot create token."); return; } bool isCustomCredentials = !string.IsNullOrWhiteSpace(_settings?.CustomSpotifyClientId) && !string.IsNullOrWhiteSpace(_settings?.CustomSpotifyClientSecret); if (isCustomCredentials) _logger.Debug("Using custom Spotify credentials from indexer settings."); else _logger.Trace("Using default shared Spotify credentials."); _tokenRequest = new(async (token) => { try { _logger.Trace("Attempting to create a new Spotify token using official endpoint."); HttpRequestMessage request = new(HttpMethod.Post, "https://accounts.spotify.com/api/token"); string credentials = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes($"{clientId}:{clientSecret}")); request.Headers.Add("Authorization", $"Basic {credentials}"); request.Content = new FormUrlEncodedContent(new Dictionary { { "grant_type", "client_credentials" } }); System.Net.Http.HttpClient httpClient = HttpGet.HttpClient; HttpResponseMessage response = await httpClient.SendAsync(request, token); response.EnsureSuccessStatusCode(); string responseContent = await response.Content.ReadAsStringAsync(token); _logger.Debug($"Spotify token created successfully"); JsonElement dynamicObject = JsonSerializer.Deserialize(responseContent)!; _token = dynamicObject.GetProperty("access_token").GetString() ?? ""; if (string.IsNullOrEmpty(_token)) return false; int expiresIn = 3600; if (dynamicObject.TryGetProperty("expires_in", out JsonElement expiresElement)) expiresIn = expiresElement.GetInt32(); _tokenExpiry = DateTime.Now.AddSeconds(expiresIn - 60); _logger.Trace($"Successfully created a new Spotify token. Expires at {_tokenExpiry}"); } catch (Exception ex) { _logger.Error(ex, "Error occurred while creating a Spotify token."); return false; } return true; }, new() { NumberOfAttempts = 1 }); } } } ================================================ FILE: Tubifarry/Indexers/Spotify/SpotifyToYouTubeEnricher.cs ================================================ using FuzzySharp; using NLog; using Tubifarry.Core.Model; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using Tubifarry.Download.Clients.YouTube; using YouTubeMusicAPI.Client; using YouTubeMusicAPI.Models.Info; using YouTubeMusicAPI.Models.Search; using YouTubeMusicAPI.Models.Streaming; using YouTubeMusicAPI.Pagination; namespace Tubifarry.Indexers.Spotify { public interface ISpotifyToYouTubeEnricher { void UpdateSettings(SpotifyIndexerSettings settings); List EnrichWithYouTubeData(List albums); Task EnrichSingleAlbumAsync(AlbumData albumData); } /// /// Enriches Spotify album data with YouTube Music streaming information. /// internal class SpotifyToYouTubeEnricher(Logger logger) : ISpotifyToYouTubeEnricher { private const int DEFAULT_BITRATE = 128; private const int MAX_CONCURRENT_ENRICHMENTS = 3; private YouTubeMusicClient? _ytClient; private SessionTokens? _sessionTokens; private readonly Logger _logger = logger; private SpotifyIndexerSettings? _currentSettings; public void UpdateSettings(SpotifyIndexerSettings settings) { if (SettingsEqual(_currentSettings, settings) && _sessionTokens?.IsValid == true) return; _currentSettings = settings; try { _sessionTokens = TrustedSessionHelper.GetTrustedSessionTokensAsync(settings.TrustedSessionGeneratorUrl).GetAwaiter().GetResult(); _ytClient = TrustedSessionHelper.CreateAuthenticatedClientAsync(settings.TrustedSessionGeneratorUrl, settings.CookiePath).GetAwaiter().GetResult(); _logger.Debug("Successfully created authenticated YouTube Music client for enrichment"); } catch (Exception ex) { _logger.Error(ex, "Failed to create authenticated YouTube Music client"); _ytClient = null; } } public List EnrichWithYouTubeData(List albums) { if (_ytClient == null) throw new NullReferenceException("YouTube client is not initialized."); List enrichedAlbums = []; foreach (AlbumData[]? batch in albums.Chunk(MAX_CONCURRENT_ENRICHMENTS).ToList()) { List> enrichmentTasks = batch.Select(album => EnrichSingleAlbumAsync(album)).Where(x => x != null).ToList()!; try { bool allCompleted = Task.WaitAll([.. enrichmentTasks], TimeSpan.FromMinutes(3)); if (!allCompleted) _logger.Warn("Some enrichment tasks timed out after 3 minutes. Proceeding with completed tasks."); foreach (Task task in enrichmentTasks) { if (task.IsCompletedSuccessfully) enrichedAlbums.Add(task.GetAwaiter().GetResult()); else if (task.IsFaulted) _logger.Warn(task.Exception, "Enrichment task failed"); } } catch (Exception ex) { _logger.Error(ex, "Unexpected error during batch enrichment"); } } _logger.Debug($"Enriched {enrichedAlbums.Count(a => a.Bitrate > DEFAULT_BITRATE)}/{albums.Count} albums with YouTube Music data"); return enrichedAlbums; } public async Task EnrichSingleAlbumAsync(AlbumData albumData) { try { if (string.IsNullOrWhiteSpace(albumData.AlbumName) || string.IsNullOrWhiteSpace(albumData.ArtistName)) { _logger.Trace($"Skipping enrichment due to missing album or artist name: '{albumData.AlbumName}' by '{albumData.ArtistName}'"); return null; } string searchQuery = $"\"{albumData.AlbumName}\" \"{albumData.ArtistName}\""; PaginatedAsyncEnumerable? searchResults = _ytClient!.SearchAsync(searchQuery, SearchCategory.Albums); if (searchResults == null) { _logger.Debug($"No search results object created for album: '{albumData.AlbumName}' by '{albumData.ArtistName}'"); return null; } int processedCount = 0; int maxSearchResults = _currentSettings?.MaxEnrichmentAttempts ?? 10; await foreach (SearchResult searchResult in searchResults) { if (searchResult is not AlbumSearchResult ytAlbum) continue; if (processedCount >= maxSearchResults) break; try { if (!IsAlbumMatch(albumData, ytAlbum)) continue; string browseId; AlbumInfo albumInfo; try { browseId = await _ytClient.GetAlbumBrowseIdAsync(ytAlbum.Id); albumInfo = await _ytClient.GetAlbumInfoAsync(browseId); } catch (Exception ex) { _logger.Debug(ex, $"Failed to get album info for YouTube album: '{ytAlbum.Name}' (ID: {ytAlbum.Id})"); continue; } if (albumInfo?.Songs == null || albumInfo.Songs.Length == 0) continue; if (albumData.TotalTracks > 0 && !IsTrackCountValid(albumData.TotalTracks, albumInfo.Songs.Length)) continue; AlbumSong? firstTrack = albumInfo.Songs.FirstOrDefault(s => !string.IsNullOrEmpty(s.Id)); if (firstTrack?.Id == null) continue; StreamingData streamingData = await _ytClient.GetStreamingDataAsync(firstTrack.Id); AudioStreamInfo? bestAudioStream = streamingData.StreamInfo .OfType() .OrderByDescending(info => info.Bitrate) .FirstOrDefault(); if (bestAudioStream != null) { albumData.AlbumId = ytAlbum.Id; albumData.Duration = (long)albumInfo.Duration.TotalSeconds; albumData.Bitrate = AudioFormatHelper.RoundToStandardBitrate(bestAudioStream.Bitrate / 1000); albumData.TotalTracks = albumInfo.SongCount; _logger.Trace($"Successfully enriched album: '{albumData.AlbumName}' (Bitrate: {albumData.Bitrate}kbps)"); return albumData; } } catch (InvalidOperationException ex) { _logger.Debug(ex, $"Failed to get streaming data for a track in album '{albumData.AlbumName}'"); } catch (Exception ex) { _logger.Debug(ex, $"Failed to process YouTube album result: '{ytAlbum?.Name ?? "Unknown"}'"); } finally { processedCount++; } } } catch (IndexOutOfRangeException) { _logger.Warn($"Search pagination failed for '{albumData.AlbumName}' by '{albumData.ArtistName}'."); } catch (Exception ex) { _logger.Error(ex, $"Unexpected error enriching album: '{albumData.AlbumName}' by '{albumData.ArtistName}'."); } return albumData; } private bool IsAlbumMatch(AlbumData spotifyAlbum, AlbumSearchResult ytAlbum) { if (string.IsNullOrEmpty(ytAlbum.Name) || string.IsNullOrEmpty(spotifyAlbum.AlbumName)) return false; string normalizedSpotifyName = NormalizeString(spotifyAlbum.AlbumName); string normalizedYtName = NormalizeString(ytAlbum.Name); bool enableFuzzyMatching = _currentSettings?.EnableFuzzyMatching ?? true; if (!AreNamesSimilar(normalizedSpotifyName, normalizedYtName, enableFuzzyMatching)) return false; if (ytAlbum.Artists?.Any() == true && !string.IsNullOrEmpty(spotifyAlbum.ArtistName)) { string normalizedSpotifyArtist = NormalizeString(spotifyAlbum.ArtistName); bool artistMatch = ytAlbum.Artists.Any(artist => AreNamesSimilar(normalizedSpotifyArtist, NormalizeString(artist.Name ?? ""), enableFuzzyMatching)); if (!artistMatch) return false; } if (ytAlbum.ReleaseYear > 0 && spotifyAlbum.ReleaseDateTime != DateTime.MinValue) { int yearTolerance = _currentSettings?.YearTolerance ?? 2; int yearDifference = Math.Abs(ytAlbum.ReleaseYear - spotifyAlbum.ReleaseDateTime.Year); if (yearDifference > yearTolerance) return false; } return true; } private bool IsTrackCountValid(int spotifyTrackCount, int ytTrackCount) { if (spotifyTrackCount <= 0 || ytTrackCount <= 0) return true; double trackCountTolerance = (_currentSettings?.TrackCountTolerance ?? 20) / 100.0; double difference = Math.Abs(spotifyTrackCount - ytTrackCount) / (double)spotifyTrackCount; return difference <= trackCountTolerance; } private static string NormalizeString(string input) { if (string.IsNullOrEmpty(input)) return string.Empty; return input.ToLowerInvariant() .Replace("&", "and") .Replace("-", " ") .Replace("_", " ") .Trim(); } private static bool AreNamesSimilar(string name1, string name2, bool enableFuzzyMatching) { if (string.IsNullOrEmpty(name1) || string.IsNullOrEmpty(name2)) return false; if (!enableFuzzyMatching) return string.Equals(name1, name2, StringComparison.OrdinalIgnoreCase); int ratio = Fuzz.Ratio(name1, name2); int partialRatio = Fuzz.PartialRatio(name1, name2); int tokenRatio = Fuzz.TokenSetRatio(name1, name2); return ratio >= 80 || partialRatio >= 85 || tokenRatio >= 85; } private static bool SettingsEqual(SpotifyIndexerSettings? settings1, SpotifyIndexerSettings? settings2) { if (settings1 == null || settings2 == null) return false; return settings1.CookiePath == settings2.CookiePath && settings1.TrustedSessionGeneratorUrl == settings2.TrustedSessionGeneratorUrl; } } } ================================================ FILE: Tubifarry/Indexers/Spotify/TubifarryIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using Tubifarry.Core.Records; using Tubifarry.Core.Replacements; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; using Tubifarry.Download.Clients.YouTube; namespace Tubifarry.Indexers.Spotify { internal class TubifarryIndexer : ExtendedHttpIndexerBase { public override string Name => "Tubifarry"; public override string Protocol => nameof(YoutubeDownloadProtocol); public override bool SupportsRss => true; public override bool SupportsSearch => true; public override int PageSize => 20; public override TimeSpan RateLimit => TimeSpan.FromSeconds(2); private readonly ISpotifyRequestGenerator _requestGenerator; private readonly ISpotifyParser _parser; public override ProviderMessage Message => new( "Spotify is used to discover music releases, but actual downloads are provided through YouTube Music. " + "This indexer searches Spotify for album information and enriches it with YouTube Music streaming data. " + "Ensure you have valid authentication for both services for the best results.", ProviderMessageType.Info ); public TubifarryIndexer( ISpotifyParser parser, ISpotifyRequestGenerator generator, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, ISentryHelper sentry, Logger logger) : base(httpClient, indexerStatusService, configService, parsingService, sentry, logger) { _parser = parser; _requestGenerator = generator; } protected override async Task Test(List failures) { UpdateComponentSettings(); if (_requestGenerator.TokenIsExpired()) _requestGenerator.StartTokenRequest(); try { await TrustedSessionHelper.ValidateAuthenticationSettingsAsync(Settings.TrustedSessionGeneratorUrl, Settings.CookiePath); SessionTokens session = await TrustedSessionHelper.GetTrustedSessionTokensAsync(Settings.TrustedSessionGeneratorUrl, true); if (!session.IsValid && !session.IsEmpty) failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", "Failed to retrieve valid tokens from the session generator service")); } catch (Exception ex) { failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", $"Failed to valiate session generator service: {ex.Message}")); } UpdateComponentSettings(); } private void UpdateComponentSettings() { _requestGenerator.UpdateSettings(Settings); _parser.UpdateSettings(Settings); } public override IIndexerRequestGenerator GetExtendedRequestGenerator() { UpdateComponentSettings(); return _requestGenerator; } public override IParseIndexerResponse GetParser() { UpdateComponentSettings(); return _parser; } } } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicAuthHelper.cs ================================================ using System.Text; namespace Tubifarry.Indexers.SubSonic { /// /// Helper class for SubSonic authentication /// Handles token generation, MD5 hashing, and URL building for secure authentication /// public static class SubSonicAuthHelper { public const string ClientName = PluginInfo.Name; public const string ApiVersion = "1.16.1"; public static (string Salt, string Token) GenerateToken(string password) { string salt = GenerateSaltFromAssembly(); string token = CalculateMd5Hash(password + salt); return (salt, token); } public static void AppendAuthParameters(StringBuilder urlBuilder, string username, string password, bool useTokenAuth) { string separator = urlBuilder.ToString().Contains('?') ? "&" : "?"; urlBuilder.Append($"{separator}u={Uri.EscapeDataString(username)}"); urlBuilder.Append($"&v={Uri.EscapeDataString(ApiVersion)}"); urlBuilder.Append($"&c={Uri.EscapeDataString(ClientName)}"); if (useTokenAuth) { (string salt, string token) = GenerateToken(password); urlBuilder.Append($"&t={token}"); urlBuilder.Append($"&s={salt}"); } else { urlBuilder.Append($"&p={Uri.EscapeDataString(password)}"); } } private static string GenerateSaltFromAssembly() => CalculateMd5Hash(PluginInfo.InformationalVersion + Tubifarry.UserAgent + Tubifarry.LastStarted)[..7]; private static string CalculateMd5Hash(string input) { byte[] inputBytes = Encoding.UTF8.GetBytes(input); byte[] hashBytes = System.Security.Cryptography.MD5.HashData(inputBytes); return Convert.ToHexString(hashBytes).ToLowerInvariant(); } } } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using System.Text; using System.Text.Json; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.SubSonic { public class SubSonicIndexer : HttpIndexerBase { private readonly ISubSonicRequestGenerator _requestGenerator; private readonly ISubSonicParser _parser; public override string Name => "SubSonic"; public override string Protocol => nameof(SubSonicDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 50; public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); public override ProviderMessage Message => new( "SubSonic provides access to your personal music server supporting Subsonic API.", ProviderMessageType.Info); public SubSonicIndexer( ISubSonicRequestGenerator requestGenerator, ISubSonicParser parser, IHttpClient httpClient, IIndexerStatusService statusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, statusService, configService, parsingService, logger) { _requestGenerator = requestGenerator; _parser = parser; } protected override async Task Test(List failures) { try { // Test connection using ping endpoint string baseUrl = Settings.BaseUrl.TrimEnd('/'); var urlBuilder = new StringBuilder($"{baseUrl}/rest/ping.view"); SubSonicAuthHelper.AppendAuthParameters(urlBuilder, Settings.Username, Settings.Password, Settings.UseTokenAuth); urlBuilder.Append("&f=json"); string testUrl = urlBuilder.ToString(); var request = new HttpRequest(testUrl) { RequestTimeout = TimeSpan.FromSeconds(Settings.RequestTimeout) }; request.Headers["User-Agent"] = Tubifarry.UserAgent; _logger.Trace("Testing SubSonic connection to: {BaseUrl}", Settings.BaseUrl); var response = await _httpClient.ExecuteAsync(request); if (!response.HasHttpError) { var responseWrapper = JsonSerializer.Deserialize( response.Content, IndexerParserHelper.StandardJsonOptions); if (responseWrapper?.SubsonicResponse != null) { var pingResponse = responseWrapper.SubsonicResponse; if (pingResponse.Status == "ok") { _logger.Debug("Successfully connected to SubSonic server as {Username}", Settings.Username); return; } else if (pingResponse.Error != null) { int errorCode = pingResponse.Error.Code; string errorMsg = pingResponse.Error.Message; if (errorCode == 40 || errorCode == 41) // Authentication errors { failures.Add(new ValidationFailure("Username", $"Authentication failed: {errorMsg}. Check your username and password.")); } else { failures.Add(new ValidationFailure("BaseUrl", $"SubSonic API error: {errorMsg}")); } return; } } } failures.Add(new ValidationFailure("BaseUrl", $"Failed to connect to SubSonic server. Check the server URL and credentials.")); } catch (Exception ex) { _logger.Error(ex, "Error testing SubSonic connection"); failures.Add(new ValidationFailure("BaseUrl", $"Error connecting to SubSonic: {ex.Message}")); } } public override IIndexerRequestGenerator GetRequestGenerator() { _requestGenerator.SetSetting(Settings); return _requestGenerator; } public override IParseIndexerResponse GetParser() { _parser.SetSettings(Settings); return _parser; } } } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicIndexerParser.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Text; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.SubSonic; public interface ISubSonicParser : IParseIndexerResponse { void SetSettings(SubSonicIndexerSettings settings); } public class SubSonicIndexerParser(Logger logger, IHttpClient httpClient) : ISubSonicParser { private const string ContentTypeSearch3 = "search3"; private const string ContentTypeSearch3WithSongs = "search3_with_songs"; private const string SubSonicStatusOk = "ok"; private const string JsonFormat = "json"; private const long DefaultMaxSongSize = 10 * 1024 * 1024; // 10MB private readonly Logger _logger = logger; private readonly IHttpClient _httpClient = httpClient; private SubSonicIndexerSettings? _settings; public void SetSettings(SubSonicIndexerSettings settings) => _settings = settings; public IList ParseResponse(IndexerResponse indexerResponse) { List releases = new(); try { string contentType = indexerResponse.Request.HttpRequest.ContentSummary; string responseContent = indexerResponse.Content; if (contentType == ContentTypeSearch3 || contentType == ContentTypeSearch3WithSongs) { bool includeSongs = contentType == ContentTypeSearch3WithSongs; ParseSearch3Response(responseContent, releases, includeSongs); } } catch (Exception ex) { _logger.Error(ex, "Error parsing SubSonic response"); } return releases; } private void ParseSearch3Response(string responseContent, List releases, bool includeSongs) { SubSonicResponseWrapper? searchWrapper = JsonSerializer.Deserialize( responseContent, IndexerParserHelper.StandardJsonOptions); if (searchWrapper?.SubsonicResponse == null) { _logger.Warn("Invalid search3 response structure"); return; } if (!ValidateResponse(searchWrapper.SubsonicResponse)) return; SubSonicSearchResponse? searchResult = searchWrapper.SubsonicResponse.SearchResult3; if (searchResult == null) { _logger.Warn("No search results in response"); return; } ProcessAlbums(searchResult.Albums, releases); if (includeSongs) ProcessSongs(searchResult.Songs, releases); } private bool ValidateResponse(SubSonicResponse response) { if (response.Status != SubSonicStatusOk) { if (response.Error != null) { _logger.Warn("SubSonic API error: {Message}", response.Error.Message); } return false; } return true; } private void ProcessAlbums(List? albums, List releases) { if (albums == null || albums.Count == 0) return; _logger.Trace("Processing {Count} albums from search3", albums.Count); foreach (SubSonicSearchAlbum albumFromSearch in albums) { try { SubSonicAlbumFull? fullAlbum = FetchFullAlbum(albumFromSearch.Id); if (fullAlbum != null) { AlbumData albumData = CreateAlbumData(fullAlbum); albumData.ParseReleaseDate(); releases.Add(albumData.ToReleaseInfo()); } } catch (Exception ex) { _logger.Warn(ex, "Failed to fetch album details for {AlbumId}: {AlbumName}", albumFromSearch.Id, albumFromSearch.Name); } } } private void ProcessSongs(List? songs, List releases) { if (songs == null || songs.Count == 0) { return; } _logger.Trace("Processing {Count} songs from search3", songs.Count); IndexerParserHelper.ProcessItems(songs, CreateTrackData, releases); } private SubSonicAlbumFull? FetchFullAlbum(string albumId) { if (_settings == null) { _logger.Error("Settings not initialized"); return null; } string url = BuildAlbumUrl(albumId); HttpRequest request = CreateHttpRequest(url); _logger.Trace("Fetching full album details: {AlbumId}", albumId); HttpResponse? response = ExecuteRequest(request); if (response == null) { return null; } return ParseAlbumResponse(response.Content, albumId); } private string BuildAlbumUrl(string albumId) { string baseUrl = _settings!.BaseUrl.TrimEnd('/'); StringBuilder urlBuilder = new($"{baseUrl}/rest/getAlbum.view"); urlBuilder.Append($"?id={Uri.EscapeDataString(albumId)}"); SubSonicAuthHelper.AppendAuthParameters(urlBuilder, _settings!.Username, _settings.Password, _settings.UseTokenAuth); urlBuilder.Append($"&f={JsonFormat}"); return urlBuilder.ToString(); } private HttpRequest CreateHttpRequest(string url) { HttpRequest request = new(url) { RequestTimeout = TimeSpan.FromSeconds(_settings!.RequestTimeout) }; request.Headers["User-Agent"] = Tubifarry.UserAgent; return request; } private HttpResponse? ExecuteRequest(HttpRequest request) { HttpResponse response = _httpClient.ExecuteAsync(request).GetAwaiter().GetResult(); if (response.HasHttpError) { _logger.Warn("HTTP error: {StatusCode}", response.StatusCode); return null; } return response; } private SubSonicAlbumFull? ParseAlbumResponse(string content, string albumId) { SubSonicAlbumResponseWrapper? albumWrapper = JsonSerializer.Deserialize( content, IndexerParserHelper.StandardJsonOptions); if (albumWrapper?.SubsonicResponse?.Status == SubSonicStatusOk && albumWrapper.SubsonicResponse.Album != null) { return albumWrapper.SubsonicResponse.Album; } if (albumWrapper?.SubsonicResponse?.Error != null) { _logger.Warn("SubSonic API error for album {AlbumId}: {Message}", albumId, albumWrapper.SubsonicResponse.Error.Message); } return null; } private AlbumData CreateAlbumData(SubSonicAlbumFull album) { if (album.Songs == null || album.Songs.Count == 0) { _logger.Warn("Album '{Name}' (ID: {Id}) has no songs", album.Name, album.Id); throw new InvalidOperationException($"Album {album.Id} has no songs"); } SubSonicSearchSong firstSong = album.Songs[0]; (AudioFormat format, int bitrate, int bitDepth) = IndexerParserHelper.GetQualityInfo( firstSong.Suffix, firstSong.ContentType, firstSong.BitRate); long totalSize = CalculateTotalAlbumSize(album.Songs, bitrate); _logger.Trace("Parsed album '{Name}' with {TrackCount} tracks, total size: {Size} bytes, format: {Format} {BitDepth}bit", album.Name, album.Songs.Count, totalSize, format, bitDepth); return new AlbumData("SubSonic", nameof(SubSonicDownloadProtocol)) { AlbumId = $"{_settings?.BaseUrl}/album/{album.Id}", AlbumName = album.Name, ArtistName = album.Artist, InfoUrl = BuildInfoUrl("album", album.Id), TotalTracks = album.SongCount > 0 ? album.SongCount : album.Songs.Count, ReleaseDate = album.YearString, ReleaseDatePrecision = album.Year.HasValue ? "year" : "day", CustomString = album.CoverArt ?? string.Empty, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = totalSize }; } private static long CalculateTotalAlbumSize(List songs, int bitrate) { long totalSize = 0; foreach (SubSonicSearchSong song in songs) { long songSize = IndexerParserHelper.EstimateSize( song.Size, song.Duration, bitrate, 1, DefaultMaxSongSize); totalSize += songSize; } return totalSize; } private AlbumData CreateTrackData(SubSonicSearchSong song) { (AudioFormat format, int bitrate, int bitDepth) = IndexerParserHelper.GetQualityInfo(song.Suffix, song.ContentType, song.BitRate); long actualSize = song.Size > 0 ? song.Size : IndexerParserHelper.EstimateSize(0, song.Duration, bitrate, 1, DefaultMaxSongSize); return new AlbumData("SubSonic", nameof(SubSonicDownloadProtocol)) { AlbumId = $"{_settings?.BaseUrl}/track/{song.Id}", AlbumName = song.DisplayAlbum, ArtistName = song.Artist, InfoUrl = BuildInfoUrl("track", song.Id), TotalTracks = 1, ReleaseDate = song.Year?.ToString() ?? DateTime.Now.Year.ToString(), ReleaseDatePrecision = "year", Duration = song.Duration, CustomString = song.CoverArt ?? string.Empty, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = actualSize }; } private string BuildInfoUrl(string type, string id) => string.IsNullOrWhiteSpace(_settings?.ExternalUrl) ? $"{_settings?.BaseUrl}://{type}/{id}" : $"{_settings.ExternalUrl.TrimEnd('/')}/rest/browse?type={type}&id={Uri.EscapeDataString(id)}"; } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; namespace Tubifarry.Indexers.SubSonic; public class SubSonicIndexerSettingsValidator : AbstractValidator { public SubSonicIndexerSettingsValidator() { // Base URL validation RuleFor(x => x.BaseUrl) .ValidRootUrl() .Must(url => !url.EndsWith('/')) .WithMessage("Server URL must not end with a slash ('/')."); // External URL validation (only if not empty) RuleFor(x => x.ExternalUrl) .Must(url => string.IsNullOrEmpty(url) || (Uri.IsWellFormedUriString(url, UriKind.Absolute) && !url.EndsWith('/'))) .WithMessage("External URL must be a valid URL and must not end with a slash ('/')."); // Username validation RuleFor(x => x.Username) .NotEmpty() .WithMessage("Username is required."); // Password validation RuleFor(x => x.Password) .NotEmpty() .WithMessage("Password is required."); // Search limit validation RuleFor(x => x.SearchLimit) .InclusiveBetween(1, 500) .WithMessage("Search limit must be between 1 and 500."); // Request timeout validation RuleFor(x => x.RequestTimeout) .InclusiveBetween(10, 300) .WithMessage("Request timeout must be between 10 and 300 seconds."); } } public class SubSonicIndexerSettings : IIndexerSettings { private static readonly SubSonicIndexerSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Server URL", Type = FieldType.Url, Placeholder = "https://music.example.com", HelpText = "URL of your SubSonic server (internal API URL).")] public string BaseUrl { get; set; } = string.Empty; [FieldDefinition(1, Label = "External URL", Type = FieldType.Url, Placeholder = "https://subsonic.example.com", HelpText = "External URL for info links and redirects. Leave empty to use Server URL.", Advanced = true)] public string? ExternalUrl { get; set; } = string.Empty; [FieldDefinition(2, Label = "Username", Type = FieldType.Textbox, HelpText = "Your SubSonic username.")] public string Username { get; set; } = string.Empty; [FieldDefinition(3, Label = "Password", Type = FieldType.Password, HelpText = "Your SubSonic password.", Privacy = PrivacyLevel.Password)] public string Password { get; set; } = string.Empty; [FieldDefinition(4, Label = "Use Token Authentication", Type = FieldType.Checkbox, HelpText = "Use secure token-based authentication (API 1.13.0+). Disable for older servers.", Advanced = true)] public bool UseTokenAuth { get; set; } = true; [FieldDefinition(5, Label = "Search Limit", Type = FieldType.Number, HelpText = "Maximum number of results to return per search.", Hidden = HiddenType.Hidden, Advanced = true)] public int SearchLimit { get; set; } = 50; [FieldDefinition(6, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for requests to SubSonic server.", Advanced = true)] public int RequestTimeout { get; set; } = 60; [FieldDefinition(7, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit.", Advanced = true)] public int? EarlyReleaseLimit { get; set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.SubSonic { /// /// Root wrapper for Subsonic API responses /// internal record SubSonicResponseWrapper( [property: JsonPropertyName("subsonic-response")] SubSonicResponse? SubsonicResponse); /// /// Main Subsonic API response structure /// internal record SubSonicResponse( [property: JsonPropertyName("status")] string Status, [property: JsonPropertyName("version")] string? Version = null, [property: JsonPropertyName("error")] SubSonicError? Error = null, [property: JsonPropertyName("searchResult3")] SubSonicSearchResponse? SearchResult3 = null); /// /// Ping response wrapper for connection testing /// internal record SubSonicPingResponse( [property: JsonPropertyName("subsonic-response")] SubSonicPingData? SubsonicResponse); /// /// Ping response data /// internal record SubSonicPingData( [property: JsonPropertyName("status")] string Status, [property: JsonPropertyName("version")] string? Version = null, [property: JsonPropertyName("error")] SubSonicError? Error = null); /// /// Error details from Subsonic API /// internal record SubSonicError( [property: JsonPropertyName("code")] int Code, [property: JsonPropertyName("message")] string Message); /// /// Search response from SubSonic API (search3 endpoint) /// internal record SubSonicSearchResponse( [property: JsonPropertyName("artist")] List? Artists, [property: JsonPropertyName("album")] List? Albums, [property: JsonPropertyName("song")] List? Songs); /// /// Artist model from SubSonic API /// internal record SubSonicSearchArtist( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("albumCount")] int AlbumCount = 0, [property: JsonPropertyName("coverArt")] string? CoverArt = null); /// /// Album model from SubSonic API (search results) /// internal record SubSonicSearchAlbum( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("artistId"), JsonConverter(typeof(StringConverter))] string? ArtistId = null, [property: JsonPropertyName("coverArt")] string? CoverArt = null, [property: JsonPropertyName("songCount")] int SongCount = 0, [property: JsonPropertyName("duration")] int Duration = 0, [property: JsonPropertyName("created"), JsonConverter(typeof(UnixTimestampConverter))] DateTime? Created = null, [property: JsonPropertyName("year")] int? Year = null, [property: JsonPropertyName("genre")] string? Genre = null) { [JsonIgnore] public string YearString => Year?.ToString() ?? Created?.Year.ToString() ?? DateTime.Now.Year.ToString(); } /// /// Full album with songs from getAlbum endpoint /// internal record SubSonicAlbumFull( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("artistId"), JsonConverter(typeof(StringConverter))] string? ArtistId, [property: JsonPropertyName("coverArt")] string? CoverArt, [property: JsonPropertyName("songCount")] int SongCount, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("created"), JsonConverter(typeof(UnixTimestampConverter))] DateTime? Created, [property: JsonPropertyName("year")] int? Year, [property: JsonPropertyName("genre")] string? Genre, [property: JsonPropertyName("song")] List? Songs) { [JsonIgnore] public string YearString => Year?.ToString() ?? Created?.Year.ToString() ?? DateTime.Now.Year.ToString(); } /// /// getAlbum response wrapper /// internal record SubSonicAlbumResponseWrapper( [property: JsonPropertyName("subsonic-response")] SubSonicItemResponse? SubsonicResponse); /// /// getSong response wrapper /// internal record SubSonicSongResponseWrapper( [property: JsonPropertyName("subsonic-response")] SubSonicItemResponse? SubsonicResponse); /// /// getSong response data /// internal record SubSonicItemResponse( [property: JsonPropertyName("status")] string Status, [property: JsonPropertyName("version")] string? Version = null, [property: JsonPropertyName("error")] SubSonicError? Error = null, [property: JsonPropertyName("song")] SubSonicSearchSong? Song = null, [property: JsonPropertyName("album")] SubSonicAlbumFull? Album = null); /// /// Song/Track model from SubSonic API /// internal record SubSonicSearchSong( [property: JsonPropertyName("id"), JsonConverter(typeof(StringConverter))] string Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("artistId"), JsonConverter(typeof(StringConverter))] string? ArtistId = null, [property: JsonPropertyName("album")] string? Album = null, [property: JsonPropertyName("albumId"), JsonConverter(typeof(StringConverter))] string? AlbumId = null, [property: JsonPropertyName("coverArt")] string? CoverArt = null, [property: JsonPropertyName("duration")] int Duration = 0, [property: JsonPropertyName("bitRate")] int BitRate = 0, [property: JsonPropertyName("track")] int? Track = null, [property: JsonPropertyName("year")] int? Year = null, [property: JsonPropertyName("genre")] string? Genre = null, [property: JsonPropertyName("size")] long Size = 0, [property: JsonPropertyName("suffix")] string? Suffix = null, [property: JsonPropertyName("contentType")] string? ContentType = null, [property: JsonPropertyName("path")] string? Path = null) { [JsonIgnore] public string DisplayAlbum => Album ?? "Unknown Album"; } } ================================================ FILE: Tubifarry/Indexers/SubSonic/SubSonicRequestGenerator.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using System.Text; namespace Tubifarry.Indexers.SubSonic { public interface ISubSonicRequestGenerator : IIndexerRequestGenerator { void SetSetting(SubSonicIndexerSettings settings); } public class SubSonicRequestGenerator(Logger logger) : ISubSonicRequestGenerator { private readonly Logger _logger = logger; private SubSonicIndexerSettings? _settings; public IndexerPageableRequestChain GetRecentRequests() => new(); public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { string query = string.Join(' ', new[] { searchCriteria.AlbumQuery, searchCriteria.ArtistQuery } .Where(s => !string.IsNullOrWhiteSpace(s))); bool isSingle = searchCriteria.Albums?.FirstOrDefault()?.AlbumReleases?.Value?.Min(r => r.TrackCount) == 1; return Generate(query, isSingle); } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) { return Generate(searchCriteria.ArtistQuery, false); } public void SetSetting(SubSonicIndexerSettings settings) => _settings = settings; private IndexerPageableRequestChain Generate(string query, bool isSingle) { IndexerPageableRequestChain chain = new(); if (string.IsNullOrWhiteSpace(query)) { _logger.Warn("Empty query, skipping search request"); return chain; } if (_settings == null) { _logger.Error("Settings not initialized"); return chain; } string baseUrl = _settings.BaseUrl.TrimEnd('/'); try { string searchUrl = BuildSearch3Url(baseUrl, query, isSingle); _logger.Trace($"Searching SubSonic: {searchUrl}"); IndexerRequest searchRequest = CreateRequest(searchUrl, isSingle ? "search3_with_songs" : "search3"); chain.Add([searchRequest]); } catch (Exception ex) { _logger.Error(ex, "Error generating SubSonic requests"); } return chain; } private string BuildSearch3Url(string baseUrl, string query, bool isSingle) { StringBuilder urlBuilder = new($"{baseUrl}/rest/search3.view"); urlBuilder.Append($"?query={Uri.EscapeDataString(query)}"); SubSonicAuthHelper.AppendAuthParameters(urlBuilder, _settings!.Username, _settings.Password, _settings.UseTokenAuth); urlBuilder.Append($"&artistCount=0"); urlBuilder.Append($"&albumCount={_settings!.SearchLimit}"); urlBuilder.Append($"&songCount={(isSingle ? _settings.SearchLimit : 0)}"); urlBuilder.Append("&f=json"); return urlBuilder.ToString(); } private IndexerRequest CreateRequest(string url, string contentType) { HttpRequest req = new(url) { RequestTimeout = TimeSpan.FromSeconds(_settings!.RequestTimeout), ContentSummary = contentType, SuppressHttpError = false, LogHttpError = true }; req.Headers["User-Agent"] = Tubifarry.UserAgent; return new IndexerRequest(req); } } } ================================================ FILE: Tubifarry/Indexers/TripleTriple/TripleTripleIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using System.Text.Json; using Tubifarry.Download.Base; namespace Tubifarry.Indexers.TripleTriple { public class TripleTripleIndexer : HttpIndexerBase { private readonly ITripleTripleRequestGenerator _requestGenerator; private readonly ITripleTripleParser _parser; private readonly IEnumerable _requestInterceptors; public override string Name => "T2Tunes"; public override string Protocol => nameof(AmazonMusicDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 50; public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); public override ProviderMessage Message => new("T2Tunes (TripleTriple) provides high-quality music downloads from Amazon Music.", ProviderMessageType.Info); public TripleTripleIndexer( ITripleTripleRequestGenerator requestGenerator, ITripleTripleParser parser, IHttpClient httpClient, IIndexerStatusService statusService, IConfigService configService, IParsingService parsingService, IEnumerable requestInterceptors, Logger logger) : base(httpClient, statusService, configService, parsingService, logger) { _requestGenerator = requestGenerator; _parser = parser; _requestInterceptors = requestInterceptors; } protected override async Task Test(List failures) { try { BaseHttpClient httpClient = new(Settings.BaseUrl.Trim(), _requestInterceptors, TimeSpan.FromSeconds(30)); string response = await httpClient.GetStringAsync("/api/status"); if (string.IsNullOrEmpty(response)) { failures.Add(new ValidationFailure("BaseUrl", "Cannot connect to T2Tunes instance: Empty response")); return; } JsonDocument doc = JsonDocument.Parse(response); if (!doc.RootElement.TryGetProperty("amazonMusic", out JsonElement statusElement) || statusElement.GetString()?.ToLower() != "up") { failures.Add(new ValidationFailure("BaseUrl", "T2Tunes Amazon Music service is not available")); return; } _logger.Debug("Successfully connected to T2Tunes, Amazon Music status: up"); } catch (Exception ex) { _logger.Error(ex, "Error connecting to T2Tunes API"); failures.Add(new ValidationFailure("BaseUrl", $"Cannot connect to T2Tunes instance: {ex.Message}")); } } public override IIndexerRequestGenerator GetRequestGenerator() { _requestGenerator.SetSetting(Settings); return _requestGenerator; } public override IParseIndexerResponse GetParser() => _parser; } } ================================================ FILE: Tubifarry/Indexers/TripleTriple/TripleTripleIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; namespace Tubifarry.Indexers.TripleTriple { public class TripleTripleIndexerSettingsValidator : AbstractValidator { public TripleTripleIndexerSettingsValidator() { RuleFor(x => x.BaseUrl) .NotEmpty().WithMessage("Base URL is required.") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Base URL must be a valid URL."); RuleFor(x => x.CountryCode) .Must(cc => Enum.IsDefined(typeof(TripleTripleCountry), cc)) .WithMessage("Invalid country code selected."); RuleFor(x => x.Codec) .Must(c => Enum.IsDefined(typeof(TripleTripleCodec), c)) .WithMessage("Invalid codec selected."); RuleFor(x => x.SearchLimit) .InclusiveBetween(1, 100).WithMessage("Search limit must be between 1 and 100."); RuleFor(x => x.RequestTimeout) .InclusiveBetween(10, 300).WithMessage("Request timeout must be between 10 and 300 seconds."); } } public class TripleTripleIndexerSettings : IIndexerSettings { private static readonly TripleTripleIndexerSettingsValidator _validator = new(); public TripleTripleIndexerSettings() { SearchLimit = 50; RequestTimeout = 60; } [FieldDefinition(0, Label = "Base URL", Type = FieldType.Textbox, HelpText = "URL of the T2Tunes API instance", Placeholder = "https://T2Tunes.site")] public string BaseUrl { get; set; } = string.Empty; [FieldDefinition(1, Label = "Country", Type = FieldType.Select, SelectOptions = typeof(TripleTripleCountry), HelpText = "Country code for Amazon Music region")] public int CountryCode { get; set; } = (int)TripleTripleCountry.US; [FieldDefinition(2, Label = "Preferred Codec", Type = FieldType.Select, SelectOptions = typeof(TripleTripleCodec), HelpText = "Audio codec preference for downloads")] public int Codec { get; set; } = (int)TripleTripleCodec.FLAC; [FieldDefinition(7, Label = "Search Limit", Type = FieldType.Number, HelpText = "Maximum number of results to return per search", Hidden = HiddenType.Hidden, Advanced = true)] public int SearchLimit { get; set; } [FieldDefinition(8, Type = FieldType.Number, Label = "Request Timeout", Unit = "seconds", HelpText = "Timeout for requests to T2Tunes API", Advanced = true)] public int RequestTimeout { get; set; } [FieldDefinition(9, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] public int? EarlyReleaseLimit { get; set; } public NzbDroneValidationResult Validate() => new(_validator.Validate(this)); } public enum TripleTripleCountry { [FieldOption(Label = "Argentina")] AR = 1, [FieldOption(Label = "Australia")] AU = 2, [FieldOption(Label = "Brazil")] BR = 3, [FieldOption(Label = "Canada")] CA = 4, [FieldOption(Label = "Costa Rica")] CR = 5, [FieldOption(Label = "Germany")] DE = 6, [FieldOption(Label = "Spain")] ES = 7, [FieldOption(Label = "France")] FR = 8, [FieldOption(Label = "United Kingdom")] GB = 9, [FieldOption(Label = "India")] IN = 10, [FieldOption(Label = "Italy")] IT = 11, [FieldOption(Label = "Japan")] JP = 12, [FieldOption(Label = "Mexico")] MX = 13, [FieldOption(Label = "New Zealand")] NZ = 14, [FieldOption(Label = "Portugal")] PT = 15, [FieldOption(Label = "United States")] US = 16 } public enum TripleTripleCodec { [FieldOption(Label = "FLAC (up to 24bit/192kHz)", Hint = "Lossless FLAC format")] FLAC = 1, [FieldOption(Label = "Opus (320kbps)", Hint = "High quality Opus codec")] OPUS = 2, [FieldOption(Label = "Dolby Digital Plus (E-AC-3)", Hint = "Dolby Digital Plus codec")] EAC3 = 3 } } ================================================ FILE: Tubifarry/Indexers/TripleTriple/TripleTripleParser.cs ================================================ using NLog; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.TripleTriple { public interface ITripleTripleParser : IParseIndexerResponse { } public class TripleTripleParser(Logger logger) : ITripleTripleParser { public IList ParseResponse(IndexerResponse indexerResponse) { List releases = []; try { bool isSingle = false; if (!string.IsNullOrEmpty(indexerResponse.Request.HttpRequest.ContentSummary)) { TripleTripleRequestData? requestData = JsonSerializer.Deserialize( indexerResponse.Request.HttpRequest.ContentSummary, IndexerParserHelper.StandardJsonOptions); isSingle = requestData?.IsSingle ?? false; } TripleTripleSearchResponse? response = JsonSerializer.Deserialize( indexerResponse.Content, IndexerParserHelper.StandardJsonOptions); if (response?.Results == null) { logger.Trace("No results found in response"); return releases; } foreach (TripleTripleResult result in response.Results) { if (result.Hits == null) continue; foreach (TripleTripleSearchHit hit in result.Hits) { TripleTripleDocument? document = hit.Document; if (document == null) continue; if (document.IsAlbum) { AlbumData albumData = CreateAlbumRelease(document); albumData.ParseReleaseDate(); releases.Add(albumData.ToReleaseInfo()); } else if (document.IsTrack && isSingle) { AlbumData trackData = CreateTrackRelease(document); trackData.ParseReleaseDate(); releases.Add(trackData.ToReleaseInfo()); } } } } catch (Exception ex) { logger.Error(ex, "Error parsing TripleTriple search response"); } return releases; } private AlbumData CreateAlbumRelease(TripleTripleDocument album) { (AudioFormat format, int bitrate, int bitDepth) = GetQualityForCodec(TripleTripleCodec.FLAC); int trackCount = album.TrackNum > 0 ? album.TrackNum : 10; long estimatedSize = IndexerParserHelper.EstimateSize(0, 0, bitrate, trackCount); return new("TripleTriple", nameof(AmazonMusicDownloadProtocol)) { AlbumId = $"album/{album.Asin}", AlbumName = album.Title, ArtistName = album.ArtistName, InfoUrl = $"https://music.amazon.com/albums/{album.Asin}", TotalTracks = trackCount, ReleaseDate = album.OriginalReleaseDate.HasValue && album.OriginalReleaseDate.Value > 0 ? DateTimeOffset.FromUnixTimeSeconds(album.OriginalReleaseDate.Value).ToString("yyyy-MM-dd") : DateTime.Now.Year.ToString(), ReleaseDatePrecision = album.OriginalReleaseDate.HasValue && album.OriginalReleaseDate.Value > 0 ? "day" : "year", CustomString = album.ArtOriginal?.Url ?? album.ArtOriginal?.ArtUrl ?? string.Empty, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = estimatedSize }; } private AlbumData CreateTrackRelease(TripleTripleDocument track) { (AudioFormat format, int bitrate, int bitDepth) = GetQualityForCodec(TripleTripleCodec.FLAC); long estimatedSize = IndexerParserHelper.EstimateSize(0, track.Duration, bitrate); return new("TripleTriple", nameof(AmazonMusicDownloadProtocol)) { AlbumId = $"track/{track.Asin}", AlbumName = track.AlbumName ?? track.Title, ArtistName = track.ArtistName, InfoUrl = $"https://music.amazon.com/tracks/{track.Asin}", TotalTracks = 1, ReleaseDate = track.OriginalReleaseDate.HasValue && track.OriginalReleaseDate.Value > 0 ? DateTimeOffset.FromUnixTimeSeconds(track.OriginalReleaseDate.Value).ToString("yyyy-MM-dd") : DateTime.Now.Year.ToString(), ReleaseDatePrecision = track.OriginalReleaseDate.HasValue && track.OriginalReleaseDate.Value > 0 ? "day" : "year", Duration = track.Duration, CustomString = track.ArtOriginal?.Url ?? track.ArtOriginal?.ArtUrl ?? string.Empty, Codec = format, Bitrate = bitrate, BitDepth = bitDepth, Size = estimatedSize }; } private static (AudioFormat Format, int Bitrate, int BitDepth) GetQualityForCodec(TripleTripleCodec codec) => codec switch { TripleTripleCodec.FLAC => (AudioFormat.FLAC, 1411, 0), TripleTripleCodec.OPUS => (AudioFormat.Opus, 320, 0), TripleTripleCodec.EAC3 => (AudioFormat.EAC3, 640, 0), _ => (AudioFormat.FLAC, 1411, 24) }; } } ================================================ FILE: Tubifarry/Indexers/TripleTriple/TripleTripleRecords.cs ================================================ using System.Text.Json.Serialization; namespace Tubifarry.Indexers.TripleTriple { public record TripleTripleStatusResponse( [property: JsonPropertyName("amazonMusic")] string AmazonMusic); public record TripleTripleSearchResponse( [property: JsonPropertyName("results")] List? Results); public record TripleTripleResult( [property: JsonPropertyName("hits")] List? Hits); public record TripleTripleSearchHit( [property: JsonPropertyName("document")] TripleTripleDocument? Document); public record TripleTripleDocument( [property: JsonPropertyName("__type")] string Type, [property: JsonPropertyName("asin")] string Asin, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("artistName")] string ArtistName, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("trackNum")] int TrackNum, [property: JsonPropertyName("discNum")] int DiscNum, [property: JsonPropertyName("albumName")] string? AlbumName = null, [property: JsonPropertyName("albumAsin")] string? AlbumAsin = null, [property: JsonPropertyName("artistAsin")] string? ArtistAsin = null, [property: JsonPropertyName("artOriginal")] TripleTripleArt? ArtOriginal = null, [property: JsonPropertyName("originalReleaseDate")] long? OriginalReleaseDate = null, [property: JsonPropertyName("primaryGenre")] string? PrimaryGenre = null, [property: JsonPropertyName("isrc")] string? Isrc = null, [property: JsonPropertyName("isMusicSubscription")] bool IsMusicSubscription = false) { public bool IsTrack => Type?.Contains("Track") == true; public bool IsAlbum => Type?.Contains("Album") == true; } public record TripleTripleArt( [property: JsonPropertyName("URL")] string Url, [property: JsonPropertyName("artUrl")] string? ArtUrl); public record TripleTripleMediaResponse( [property: JsonPropertyName("asin")] string Asin, [property: JsonPropertyName("stremeable")] bool? StremeableTypo, // API typo version [property: JsonPropertyName("streamable")] bool? StreamableCorrect, // Correct spelling [property: JsonPropertyName("tags")] TripleTripleTags? Tags, [property: JsonPropertyName("templateCoverUrl")] string? TemplateCoverUrl, [property: JsonPropertyName("streamInfo")] TripleTripleStreamInfo? StreamInfo, [property: JsonPropertyName("lyrics")] TripleTripleLyrics? Lyrics, [property: JsonPropertyName("decryptionKey")] string? DecryptionKey) { [JsonIgnore] public bool Streamable => StreamableCorrect ?? StremeableTypo ?? false; }; public record TripleTripleTags( [property: JsonPropertyName("album")] string? Album, [property: JsonPropertyName("albumArtist")] string? AlbumArtist, [property: JsonPropertyName("artist")] string? Artist, [property: JsonPropertyName("composer")] string? Composer, [property: JsonPropertyName("copyright")] string? Copyright, [property: JsonPropertyName("date")] string? Date, [property: JsonPropertyName("genre")] string? Genre, [property: JsonPropertyName("isrc")] string? Isrc, [property: JsonPropertyName("label")] string? Label, [property: JsonPropertyName("lyrics")] string? PlainLyrics, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("url")] string? Url, [property: JsonPropertyName("disc")] int Disc = 1, [property: JsonPropertyName("discTotal")] int DiscTotal = 1, [property: JsonPropertyName("track")] int Track = 0, [property: JsonPropertyName("trackTotal")] int TrackTotal = 0); public record TripleTripleStreamInfo( [property: JsonPropertyName("format")] string Format, [property: JsonPropertyName("streamUrl")] string StreamUrl, [property: JsonPropertyName("pssh")] string? Pssh, [property: JsonPropertyName("kid")] string? Kid, [property: JsonPropertyName("codec")] string Codec, [property: JsonPropertyName("sampleRate")] int SampleRate); public record TripleTripleLyrics( [property: JsonPropertyName("synced")] string? Synced, [property: JsonPropertyName("unsynced")] string? Unsynced); public record TripleTripleAlbumMetadata( [property: JsonPropertyName("albumList")] List? AlbumList); public record TripleTripleAlbumInfo( [property: JsonPropertyName("asin")] string Asin, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("artist")] TripleTripleArtistInfo Artist, [property: JsonPropertyName("image")] string? Image, [property: JsonPropertyName("label")] string? Label, [property: JsonPropertyName("originalReleaseDate")] long OriginalReleaseDate, [property: JsonPropertyName("merchantReleaseDate")] long? MerchantReleaseDate, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("trackCount")] int TrackCount, [property: JsonPropertyName("isMusicSubscription")] bool IsMusicSubscription, [property: JsonPropertyName("primaryGenreName")] string? PrimaryGenreName, [property: JsonPropertyName("tracks")] List? Tracks); public record TripleTripleArtistInfo( [property: JsonPropertyName("asin")] string Asin, [property: JsonPropertyName("name")] string Name); public record TripleTripleTrackInfo( [property: JsonPropertyName("asin")] string Asin, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("trackNum")] int TrackNum, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("isrc")] string? Isrc, [property: JsonPropertyName("songWriters")] List? SongWriters, [property: JsonPropertyName("lyrics")] TripleTripleLyrics? Lyrics, [property: JsonPropertyName("streamInfo")] TripleTripleStreamInfo? StreamInfo); public record TripleTripleRequestData( [property: JsonPropertyName("baseUrl")] string BaseUrl, [property: JsonPropertyName("country")] string Country, [property: JsonPropertyName("codec")] string Codec, [property: JsonPropertyName("isSingle")] bool IsSingle); } ================================================ FILE: Tubifarry/Indexers/TripleTriple/TripleTripleRequestGenerator.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; namespace Tubifarry.Indexers.TripleTriple { public interface ITripleTripleRequestGenerator : IIndexerRequestGenerator { void SetSetting(TripleTripleIndexerSettings settings); } public class TripleTripleRequestGenerator : ITripleTripleRequestGenerator { private readonly Logger _logger; private TripleTripleIndexerSettings? _settings; public TripleTripleRequestGenerator(Logger logger) => _logger = logger; public IndexerPageableRequestChain GetRecentRequests() => new(); public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { string query = string.Join(' ', new[] { searchCriteria.AlbumQuery, searchCriteria.ArtistQuery }.Where(s => !string.IsNullOrWhiteSpace(s))); bool isSingle = searchCriteria.Albums?.FirstOrDefault()?.AlbumReleases?.Value?.Min(r => r.TrackCount) == 1; return Generate(query, isSingle); } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) => Generate(searchCriteria.ArtistQuery, false); public void SetSetting(TripleTripleIndexerSettings settings) => _settings = settings; private IndexerPageableRequestChain Generate(string query, bool isSingle) { IndexerPageableRequestChain chain = new(); if (string.IsNullOrWhiteSpace(query)) { _logger.Warn("Empty query, skipping search request"); return chain; } string baseUrl = _settings!.BaseUrl.TrimEnd('/'); string country = ((TripleTripleCountry)_settings.CountryCode).ToString(); string codec = ((TripleTripleCodec)_settings.Codec).ToString().ToLowerInvariant(); string url = $"{baseUrl}/api/amazon-music/search?query={Uri.EscapeDataString(query)}&types=track,album&country={country}"; _logger.Trace("Creating TripleTriple search request: {Url}", url); HttpRequest req = new(url) { RequestTimeout = TimeSpan.FromSeconds(_settings.RequestTimeout), ContentSummary = new TripleTripleRequestData(baseUrl, country, codec, isSingle).ToJson() }; req.Headers["User-Agent"] = Tubifarry.UserAgent; req.Headers["Referer"] = $"{baseUrl}/search/{Uri.EscapeDataString(query)}"; chain.AddTier([new IndexerRequest(req)]); if (isSingle) { string fallbackUrl = $"{baseUrl}/api/amazon-music/search?query={Uri.EscapeDataString(query)}&types=track&country={country}"; _logger.Trace("Adding fallback track-only search: {Url}", fallbackUrl); HttpRequest fallbackReq = new(fallbackUrl) { RequestTimeout = TimeSpan.FromSeconds(_settings.RequestTimeout), ContentSummary = new TripleTripleRequestData(baseUrl, country, codec, true).ToJson() }; fallbackReq.Headers["User-Agent"] = Tubifarry.UserAgent; fallbackReq.Headers["Referer"] = $"{baseUrl}/search/{Uri.EscapeDataString(query)}"; chain.AddTier([new IndexerRequest(fallbackReq)]); } return chain; } } } ================================================ FILE: Tubifarry/Indexers/YouTube/YoutubeIndexer.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; using Tubifarry.Core.Records; using Tubifarry.Core.Replacements; using Tubifarry.Core.Telemetry; using Tubifarry.Core.Utilities; using Tubifarry.Download.Clients.YouTube; namespace Tubifarry.Indexers.YouTube { internal class YouTubeIndexer : ExtendedHttpIndexerBase { public override string Name => "Youtube"; public override string Protocol => nameof(YoutubeDownloadProtocol); public override bool SupportsRss => false; public override bool SupportsSearch => true; public override int PageSize => 50; public new YouTubeIndexerSettings Settings => base.Settings; public override TimeSpan RateLimit => TimeSpan.FromSeconds(2); private readonly YouTubeRequestGenerator _requestGenerator; private readonly YouTubeParser _parser; public override ProviderMessage Message => new( "YouTube frequently blocks downloads to prevent unauthorized access. To confirm you're not a bot, you may need to provide additional verification. " + "This issue can often be partially resolved by using a `cookies.txt` file containing your login tokens. " + "Ensure the file is properly formatted and includes valid session data to bypass restrictions. " + "Note: YouTube does not always provide the best metadata for tracks, so you may need to manually verify or update track information.", ProviderMessageType.Warning ); public YouTubeIndexer(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, ISentryHelper sentry, Logger logger) : base(httpClient, indexerStatusService, configService, parsingService, sentry, logger) { _parser = new(this); _requestGenerator = new(this); } protected override async Task Test(List failures) { try { await TrustedSessionHelper.ValidateAuthenticationSettingsAsync(Settings.TrustedSessionGeneratorUrl, Settings.CookiePath); SessionTokens session = await TrustedSessionHelper.GetTrustedSessionTokensAsync(Settings.TrustedSessionGeneratorUrl, true); if (!session.IsValid && !session.IsEmpty) failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", "Failed to retrieve valid tokens from the session generator service")); } catch (Exception ex) { failures.Add(new ValidationFailure("TrustedSessionGeneratorUrl", $"Failed to valiate session generator service: {ex.Message}")); } } public override IIndexerRequestGenerator GetExtendedRequestGenerator() => _requestGenerator; public override IParseIndexerResponse GetParser() => _parser; } } ================================================ FILE: Tubifarry/Indexers/YouTube/YoutubeIndexerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Indexers.YouTube { public class YouTubeIndexerSettingsValidator : AbstractValidator { public YouTubeIndexerSettingsValidator() { // Validate CookiePath (if provided) RuleFor(x => x.CookiePath) .Must(path => string.IsNullOrEmpty(path) || File.Exists(path)) .WithMessage("Cookie file does not exist. Please provide a valid path to the cookies file.") .Must(path => string.IsNullOrEmpty(path) || CookieManager.ParseCookieFile(path).Length != 0) .WithMessage("Cookie file is invalid or contains no valid cookies."); // Validate TrustedSessionGeneratorUrl (optional) RuleFor(x => x.TrustedSessionGeneratorUrl) .Must(url => string.IsNullOrEmpty(url) || Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("Trusted Session Generator URL must be a valid URL if provided."); } } public class YouTubeIndexerSettings : IIndexerSettings { private static readonly YouTubeIndexerSettingsValidator Validator = new(); [FieldDefinition(0, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] public int? EarlyReleaseLimit { get; set; } = null; [FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Hidden = HiddenType.Visible, Placeholder = "/path/to/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but helps with accessing restricted content.", Advanced = true)] public string CookiePath { get; set; } = string.Empty; [FieldDefinition(3, Label = "Generator URL", Type = FieldType.Textbox, Placeholder = "http://localhost:8080", HelpText = "URL to the YouTube Trusted Session Generator service. When provided, PoToken and Visitor Data will be fetched automatically.", Advanced = true)] public string TrustedSessionGeneratorUrl { get; set; } = string.Empty; public string BaseUrl { get; set; } = string.Empty; public virtual NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Indexers/YouTube/YoutubeParser.cs ================================================ using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using System.Reflection; using Tubifarry.Core.Model; using Tubifarry.Core.Records; using Tubifarry.Core.Utilities; using Tubifarry.Download.Clients.YouTube; using YouTubeMusicAPI.Client; using YouTubeMusicAPI.Models.Info; using YouTubeMusicAPI.Models.Search; using YouTubeMusicAPI.Models.Streaming; using YouTubeMusicAPI.Pagination; namespace Tubifarry.Indexers.YouTube { /// /// Parses YouTube Music API responses and converts them to releases. /// internal class YouTubeParser : IParseIndexerResponse { private const int DEFAULT_BITRATE = 128; private readonly Logger _logger; private readonly YouTubeIndexer _youTubeIndexer; private YouTubeMusicClient? _youTubeClient; private SessionTokens? _sessionToken; private static readonly Lazy?>?> _getPageDelegate = new(() => { try { Assembly ytMusicAssembly = typeof(YouTubeMusicClient).Assembly; Type? searchParserType = ytMusicAssembly.GetType("YouTubeMusicAPI.Internal.Parsers.SearchParser"); MethodInfo? getPageMethod = searchParserType?.GetMethod("GetPage", BindingFlags.Public | BindingFlags.Static); if (getPageMethod == null) return null; return (Func?>)Delegate.CreateDelegate( typeof(Func?>), getPageMethod); } catch { return null; } }); public YouTubeParser(YouTubeIndexer indexer) { _youTubeIndexer = indexer; _logger = NzbDroneLogger.GetLogger(this); } public IList ParseResponse(IndexerResponse indexerResponse) { List releases = []; try { if (string.IsNullOrEmpty(indexerResponse.Content)) { _logger.Warn("Received empty response content"); return releases; } JObject jsonResponse = JObject.Parse(indexerResponse.Content); Page searchPage = TryParseWithDelegate(jsonResponse) ?? new Page([], null); _logger.Trace($"Parsed {searchPage.Items.Count} search results from YouTube Music API response"); ProcessSearchResults(searchPage.Items, releases); _logger.Debug($"Successfully converted {releases.Count} results to releases"); return [.. releases.DistinctBy(x => x.DownloadUrl).OrderByDescending(o => o.PublishDate)]; } catch (Exception ex) { _logger.Error(ex, $"An error occurred while parsing YouTube Music API response. Response length: {indexerResponse.Content?.Length ?? 0}"); return releases; } } /// /// Try to parse using cached delegate to access internal SearchParser - 50x faster than reflection /// private Page? TryParseWithDelegate(JObject jsonResponse) { try { Func?>? delegateMethod = _getPageDelegate.Value; if (delegateMethod == null) { _logger.Error("SearchParser.GetPage delegate not available"); return null; } Page? result = delegateMethod(jsonResponse); if (result != null) { _logger.Trace("Successfully parsed response using cached delegate"); return result; } } catch (Exception ex) { _logger.Debug(ex, "Failed to parse response using delegate, falling back to manual parsing"); } return null; } private void ProcessSearchResults(IReadOnlyList searchResults, List releases) { foreach (SearchResult searchResult in searchResults) { if (searchResult is not AlbumSearchResult album) continue; try { AlbumData albumData = ExtractAlbumInfo(album); albumData.ParseReleaseDate(); EnrichAlbumWithYouTubeDataAsync(albumData).GetAwaiter().GetResult(); if (albumData.Bitrate > 0) { releases.Add(albumData.ToReleaseInfo()); _logger.Trace($"Added album: '{albumData.AlbumName}' by '{albumData.ArtistName}' (Bitrate: {albumData.Bitrate}kbps)"); } else { _logger.Trace($"Skipped album (no bitrate): '{albumData.AlbumName}' by '{albumData.ArtistName}'"); } } catch (Exception ex) { _logger.Error(ex, $"Failed to process album: '{album?.Name}' by '{album?.Artists?.FirstOrDefault()?.Name}'"); } } } private async Task EnrichAlbumWithYouTubeDataAsync(AlbumData albumData) { try { UpdateClient(); string browseId = await _youTubeClient!.GetAlbumBrowseIdAsync(albumData.AlbumId); AlbumInfo albumInfo = await _youTubeClient.GetAlbumInfoAsync(browseId); if (albumInfo?.Songs == null || albumInfo.Songs.Length == 0) { _logger.Trace($"No songs found for album: '{albumData.AlbumName}'"); albumData.Bitrate = DEFAULT_BITRATE; return; } albumData.Duration = (long)albumInfo.Duration.TotalSeconds; albumData.TotalTracks = albumInfo.SongCount; albumData.ExplicitContent = albumInfo.Songs.Any(x => x.IsExplicit); AlbumSong? firstTrack = albumInfo.Songs.FirstOrDefault(s => !string.IsNullOrEmpty(s.Id)); if (firstTrack?.Id != null) { try { StreamingData streamingData = await _youTubeClient.GetStreamingDataAsync(firstTrack.Id); AudioStreamInfo? highestQualityStream = streamingData.StreamInfo .OfType() .OrderByDescending(info => info.Bitrate) .FirstOrDefault(); if (highestQualityStream != null) albumData.Bitrate = AudioFormatHelper.RoundToStandardBitrate(highestQualityStream.Bitrate / 1000); else albumData.Bitrate = DEFAULT_BITRATE; } catch (Exception ex) { _logger.Debug(ex, $"Failed to get streaming data for track '{firstTrack.Name}' in album '{albumData.AlbumName}'"); albumData.Bitrate = DEFAULT_BITRATE; } } else { albumData.Bitrate = DEFAULT_BITRATE; } } catch (Exception ex) { _logger.Debug(ex, $"Failed to enrich album data for: '{albumData.AlbumName}'"); albumData.Bitrate = DEFAULT_BITRATE; } } private void UpdateClient() { if (_sessionToken?.IsValid == true) return; _sessionToken = TrustedSessionHelper.GetTrustedSessionTokensAsync(_youTubeIndexer.Settings.TrustedSessionGeneratorUrl).GetAwaiter().GetResult(); _youTubeClient = TrustedSessionHelper.CreateAuthenticatedClientAsync(_youTubeIndexer.Settings.TrustedSessionGeneratorUrl, _youTubeIndexer.Settings.CookiePath).GetAwaiter().GetResult(); } private static AlbumData ExtractAlbumInfo(AlbumSearchResult album) => new("Youtube", nameof(YoutubeDownloadProtocol)) { AlbumId = album.Id, InfoUrl = $"https://music.youtube.com/playlist?list={album.Id}", AlbumName = album.Name, ArtistName = album.Artists.FirstOrDefault()?.Name ?? "Unknown Artist", ReleaseDate = album.ReleaseYear > 0 ? album.ReleaseYear.ToString() : "0000-01-01", ReleaseDatePrecision = "year", CustomString = album.Thumbnails.FirstOrDefault()?.Url ?? string.Empty, CoverResolution = album.Thumbnails.FirstOrDefault() is { } thumbnail ? $"{thumbnail.Width}x{thumbnail.Height}" : "Unknown Resolution" }; } } ================================================ FILE: Tubifarry/Indexers/YouTube/YoutubeRequestGenerator.cs ================================================ using Newtonsoft.Json; using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using System.Net; using Tubifarry.Core.Records; using Tubifarry.Core.Replacements; using Tubifarry.Core.Utilities; using Tubifarry.Download.Clients.YouTube; using YouTubeMusicAPI.Internal; using YouTubeMusicAPI.Models.Search; namespace Tubifarry.Indexers.YouTube { internal class YouTubeRequestGenerator : IIndexerRequestGenerator { private const int MaxPages = 3; private readonly Logger _logger; private readonly YouTubeIndexer _youTubeIndexer; private SessionTokens? _sessionToken; public YouTubeRequestGenerator(YouTubeIndexer indexer) { _youTubeIndexer = indexer; _logger = NzbDroneLogger.GetLogger(this); } public IndexerPageableRequestChain GetRecentRequests() { // YouTube doesn't support RSS/recent releases functionality in a traditional sense return new LazyIndexerPageableRequestChain(); } public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) { _logger.Debug($"Generating search requests for album: '{searchCriteria.AlbumQuery}' by artist: '{searchCriteria.ArtistQuery}'"); LazyIndexerPageableRequestChain chain = new(5); // Primary search: album + artist if (!string.IsNullOrEmpty(searchCriteria.AlbumQuery) && !string.IsNullOrEmpty(searchCriteria.ArtistQuery)) { string primaryQuery = $"{searchCriteria.AlbumQuery} {searchCriteria.ArtistQuery}"; chain.AddFactory(() => GetRequests(primaryQuery, SearchCategory.Albums)); } // Fallback search: album only if (!string.IsNullOrEmpty(searchCriteria.AlbumQuery)) { chain.AddTierFactory(() => GetRequests(searchCriteria.AlbumQuery, SearchCategory.Albums)); } // Last resort: artist only (still search for albums) if (!string.IsNullOrEmpty(searchCriteria.ArtistQuery)) { chain.AddTierFactory(() => GetRequests(searchCriteria.ArtistQuery, SearchCategory.Albums)); } return chain; } public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) { _logger.Debug($"Generating search requests for artist: '{searchCriteria.ArtistQuery}'"); LazyIndexerPageableRequestChain chain = new(5); if (!string.IsNullOrEmpty(searchCriteria.ArtistQuery)) chain.AddFactory(() => GetRequests(searchCriteria.ArtistQuery, SearchCategory.Albums)); return chain; } private void UpdateTokens() { if (_sessionToken?.IsValid == true) return; _sessionToken = TrustedSessionHelper.GetTrustedSessionTokensAsync(_youTubeIndexer.Settings.TrustedSessionGeneratorUrl).GetAwaiter().GetResult(); } private IEnumerable GetRequests(string searchQuery, SearchCategory category) { UpdateTokens(); for (int page = 0; page < MaxPages; page++) { Dictionary payload = Payload.WebRemix( geographicalLocation: "US", visitorData: _sessionToken!.VisitorData, poToken: _sessionToken!.PoToken, signatureTimestamp: null, items: [ ("query", searchQuery), ("params", ToParams(category)), ("continuation", null) ] ); string jsonPayload = JsonConvert.SerializeObject(payload, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); HttpRequest request = new($"https://music.youtube.com/youtubei/v1/search?key={PluginKeys.YouTubeSecret}", HttpAccept.Json) { Method = HttpMethod.Post }; if (!string.IsNullOrEmpty(_youTubeIndexer.Settings.CookiePath)) { try { foreach (Cookie cookie in CookieManager.ParseCookieFile(_youTubeIndexer.Settings.CookiePath)) request.Cookies[cookie.Name] = cookie.Value; } catch (Exception ex) { _logger.Warn(ex, $"Failed to load cookies from {_youTubeIndexer.Settings.CookiePath}"); } } request.SetContent(jsonPayload); _logger.Trace($"Created YouTube Music API request for query: '{searchQuery}', category: {category}"); yield return new IndexerRequest(request); } } public static string? ToParams(SearchCategory? value) => value switch { SearchCategory.Songs => "EgWKAQIIAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", SearchCategory.Videos => "EgWKAQIQAWoQEAMQBBAJEAoQBRAREBAQFQ%3D%3D", SearchCategory.Albums => "EgWKAQIYAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", SearchCategory.CommunityPlaylists => "EgeKAQQoAEABahAQAxAKEAkQBBAFEBEQEBAV", SearchCategory.Artists => "EgWKAQIgAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", SearchCategory.Podcasts => "EgWKAQJQAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", SearchCategory.Episodes => "EgWKAQJIAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", SearchCategory.Profiles => "EgWKAQJYAWoQEAMQChAJEAQQBRAREBAQFQ%3D%3D", _ => null }; } } ================================================ FILE: Tubifarry/Metadata/Converter/AudioConverter.cs ================================================ using NLog; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Extras.Metadata.Files; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Music; using NzbDrone.Core.Tags; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; using Xabe.FFmpeg; namespace Tubifarry.Metadata.Converter { public class AudioConverter(Logger logger, Lazy tagService) : MetadataBase { private readonly Logger _logger = logger; private readonly Lazy _tagService = tagService; public override string Name => "Codec Tinker"; public override MetadataFile FindMetadataFile(Artist artist, string path) => default!; public override MetadataFileResult ArtistMetadata(Artist artist) => default!; public override MetadataFileResult AlbumMetadata(Artist artist, Album album, string albumPath) => default!; public override List ArtistImages(Artist artist) => default!; public override List AlbumImages(Artist artist, Album album, string albumFolder) => default!; public override List TrackImages(Artist artist, TrackFile trackFile) => default!; public override MetadataFileResult TrackMetadata(Artist artist, TrackFile trackFile) { if (ShouldConvertTrack(trackFile).GetAwaiter().GetResult()) ConvertTrack(trackFile).GetAwaiter().GetResult(); else _logger.Trace($"No rule matched for {trackFile.OriginalFilePath}"); return null!; } private async Task ConvertTrack(TrackFile trackFile) { AudioFormat trackFormat = await GetTrackAudioFormatAsync(trackFile.Path); if (trackFormat == AudioFormat.Unknown) return; int? currentBitrate = await GetTrackBitrateAsync(trackFile.Path); ConversionResult result = await GetTargetConversionForTrack(trackFormat, currentBitrate, trackFile); if (result.IsBlocked) return; LogConversionPlan(trackFormat, currentBitrate, result.TargetFormat, result.TargetBitrate, trackFile.Path); await PerformConversion(trackFile, result); } private async Task PerformConversion(TrackFile trackFile, ConversionResult result) { AudioMetadataHandler audioHandler = new(trackFile.Path); bool success = await audioHandler.TryConvertToFormatAsync(result.TargetFormat, result.TargetBitrate, result.TargetBitDepth, result.UseCBR); trackFile.Path = audioHandler.TrackPath; if (success) _logger.Info($"Successfully converted track: {trackFile.Path}"); else _logger.Warn($"Failed to convert track: {trackFile.Path}"); } private async Task GetTrackBitrateAsync(string filePath) { try { IMediaInfo mediaInfo = await FFmpeg.GetMediaInfo(filePath); IAudioStream? audioStream = mediaInfo.AudioStreams.FirstOrDefault(); if (audioStream == null) return null; return AudioFormatHelper.RoundToStandardBitrate((int)(audioStream.Bitrate / 1000)); } catch (Exception ex) { _logger.Error(ex, "Failed to get bitrate: {0}", filePath); return null; } } private async Task GetTrackBitDepthAsync(string filePath) { try { string probeArgs = $"-v error -select_streams a:0 -show_entries stream=bits_per_raw_sample,sample_fmt -of default=noprint_wrappers=1 \"{filePath}\""; string probeOutput = await Probe.New().Start(probeArgs); if (string.IsNullOrWhiteSpace(probeOutput)) return null; foreach (string line in probeOutput.Split('\n', StringSplitOptions.RemoveEmptyEntries)) { if (line.StartsWith("bits_per_raw_sample=") && int.TryParse(line.AsSpan(20), out int bitsPerRaw) && bitsPerRaw > 0) return bitsPerRaw; } foreach (string line in probeOutput.Split('\n', StringSplitOptions.RemoveEmptyEntries)) { if (line.StartsWith("sample_fmt=")) { string sampleFmt = line[11..].Trim().ToLower(); return sampleFmt switch { "s16" or "s16le" or "s16be" or "s16p" => 16, "s24" or "s24le" or "s24be" or "s24p" => 24, "s32" or "s32le" or "s32be" or "s32p" => 32, _ => null }; } } return null; } catch (Exception ex) { _logger.Error(ex, "Failed to get bit depth: {0}", filePath); return null; } } private ConversionResult ShouldBlockConversion(ConversionRule rule, AudioFormat trackFormat, int? currentBitrate, int? currentBitDepth) { if (rule.TargetFormat == AudioFormat.Unknown) return ConversionResult.Blocked(); // Block lossy to lossless conversion if (AudioFormatHelper.IsLossyFormat(trackFormat) && !AudioFormatHelper.IsLossyFormat(rule.TargetFormat)) { _logger.Warn($"Blocked lossy to lossless conversion from {trackFormat} to {rule.TargetFormat}"); return ConversionResult.Blocked(); } // Block bitrate upsampling for lossy formats if (AudioFormatHelper.IsLossyFormat(trackFormat) && AudioFormatHelper.IsLossyFormat(rule.TargetFormat) && currentBitrate.HasValue && rule.TargetBitrate.HasValue && rule.TargetBitrate.Value > currentBitrate.Value) { _logger.Warn($"Blocked bitrate upsampling from {currentBitrate}kbps to {rule.TargetBitrate}kbps for {trackFormat}"); return ConversionResult.Blocked(); } // Block bit depth upsampling for lossless formats if (!AudioFormatHelper.IsLossyFormat(trackFormat) && !AudioFormatHelper.IsLossyFormat(rule.TargetFormat) && currentBitDepth.HasValue && rule.TargetBitDepth.HasValue && rule.TargetBitDepth.Value > currentBitDepth.Value) { _logger.Warn($"Blocked bit depth upsampling from {currentBitDepth}-bit to {rule.TargetBitDepth}-bit for {trackFormat}"); return ConversionResult.Blocked(); } return ConversionResult.FromRule(rule); } private async Task GetTargetConversionForTrack(AudioFormat trackFormat, int? currentBitrate, TrackFile trackFile) { int? currentBitDepth = null; // Get current bit depth for lossless formats if (!AudioFormatHelper.IsLossyFormat(trackFormat)) { currentBitDepth = await GetTrackBitDepthAsync(trackFile.Path); } // Check artist tag rule first ConversionRule? artistRule = GetArtistTagRule(trackFile); if (artistRule != null) { ConversionResult result = ShouldBlockConversion(artistRule, trackFormat, currentBitrate, currentBitDepth); if (result.IsBlocked) return result; _logger.Debug($"Using artist tag rule for {trackFile.Artist?.Value?.Name}: {artistRule.TargetFormat}" + (artistRule.TargetBitrate.HasValue ? $":{artistRule.TargetBitrate}kbps" : artistRule.TargetBitDepth.HasValue ? $":{artistRule.TargetBitDepth}-bit" : "") + (artistRule.UseCBR ? ":cbr" : "")); return result; } // Check custom conversion rules foreach (KeyValuePair ruleEntry in Settings.CustomConversion) { if (!RuleParser.TryParseRule(ruleEntry.Key, ruleEntry.Value, out ConversionRule rule)) continue; if (!IsRuleMatching(rule, trackFormat, currentBitrate)) continue; ConversionResult result = ShouldBlockConversion(rule, trackFormat, currentBitrate, currentBitDepth); if (result.IsBlocked) return result; return result; } return ConversionResult.Success((AudioFormat)Settings.TargetFormat); } private async Task ShouldConvertTrack(TrackFile trackFile) { ConversionRule? artistRule = GetArtistTagRule(trackFile); if (artistRule != null && artistRule.TargetFormat == AudioFormat.Unknown) { _logger.Debug($"Skipping conversion due to no-conversion artist tag for {trackFile.Artist?.Value?.Name}"); return false; } AudioFormat trackFormat = await GetTrackAudioFormatAsync(trackFile.Path); if (trackFormat == AudioFormat.Unknown) return false; int? currentBitrate = await GetTrackBitrateAsync(trackFile.Path); _logger.Trace($"Track bitrate found for {trackFile.Path} at {currentBitrate ?? 0}kbps"); if (artistRule != null) return true; if (MatchesAnyCustomRule(trackFormat, currentBitrate)) return true; return IsFormatEnabledForConversion(trackFormat); } private ConversionRule? GetArtistTagRule(TrackFile trackFile) { if (trackFile.Artist?.Value?.Tags == null || trackFile.Artist.Value.Tags.Count == 0) return null; foreach (Tag? tag in trackFile.Artist.Value.Tags.Select(x => _tagService.Value.GetTag(x))) { if (RuleParser.TryParseArtistTag(tag.Label, out ConversionRule rule)) { _logger.Debug($"Found artist tag rule: {tag.Label} for {trackFile.Artist.Value.Name}"); return rule; } } return null; } private bool MatchesAnyCustomRule(AudioFormat trackFormat, int? currentBitrate) => Settings.CustomConversion.Any(ruleEntry => RuleParser.TryParseRule(ruleEntry.Key, ruleEntry.Value, out ConversionRule rule) && IsRuleMatching(rule, trackFormat, currentBitrate)); private bool IsRuleMatching(ConversionRule rule, AudioFormat trackFormat, int? currentBitrate) { bool formatMatches = rule.MatchesFormat(trackFormat); bool bitrateMatches = rule.MatchesBitrate(currentBitrate); if (formatMatches && bitrateMatches) { _logger.Debug($"Matched conversion rule: {rule}"); return true; } return false; } private async Task GetTrackAudioFormatAsync(string trackPath) { string extension = Path.GetExtension(trackPath); // For .m4a files, use codec detection since they can contain AAC or ALAC if (string.Equals(extension, ".m4a", StringComparison.OrdinalIgnoreCase)) { AudioFormat detectedFormat = await AudioMetadataHandler.GetSupportedCodecAsync(trackPath); if (detectedFormat != AudioFormat.Unknown) { _logger.Trace($"Detected codec-based format {detectedFormat} for .m4a file: {trackPath}"); return detectedFormat; } _logger.Warn($"Failed to detect codec for .m4a file, falling back to extension-based detection: {trackPath}"); } // For all other extensions, use extension-based detection AudioFormat trackFormat = AudioFormatHelper.GetAudioCodecFromExtension(extension); if (trackFormat == AudioFormat.Unknown) _logger.Warn($"Unknown audio format for track: {trackPath}"); return trackFormat; } private void LogConversionPlan(AudioFormat sourceFormat, int? sourceBitrate, AudioFormat targetFormat, int? targetBitrate, string trackPath) { string sourceDescription = FormatDescriptionWithBitrate(sourceFormat, sourceBitrate); string targetDescription = FormatDescriptionWithBitrate(targetFormat, targetBitrate); _logger.Debug($"Converting {sourceDescription} to {targetDescription}: {trackPath}"); } private static string FormatDescriptionWithBitrate(AudioFormat format, int? bitrate) => format + (bitrate.HasValue ? $" ({bitrate}kbps)" : ""); private bool IsFormatEnabledForConversion(AudioFormat format) => format switch { AudioFormat.MP3 => Settings.ConvertMP3, AudioFormat.AAC => Settings.ConvertAAC, AudioFormat.FLAC => Settings.ConvertFLAC, AudioFormat.WAV => Settings.ConvertWAV, AudioFormat.Opus => Settings.ConvertOpus, AudioFormat.APE => Settings.ConvertOther, AudioFormat.Vorbis => Settings.ConvertOther, AudioFormat.OGG => Settings.ConvertOther, AudioFormat.WMA => Settings.ConvertOther, AudioFormat.ALAC => Settings.ConvertOther, AudioFormat.AIFF => Settings.ConvertOther, AudioFormat.AMR => Settings.ConvertOther, _ => false }; } } ================================================ FILE: Tubifarry/Metadata/Converter/AudioConverterSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; using Xabe.FFmpeg; namespace Tubifarry.Metadata.Converter { public class AudioConverterSettingsValidator : AbstractValidator { public AudioConverterSettingsValidator() { // Validate FFmpegPath RuleFor(x => x.FFmpegPath) .NotEmpty() .WithMessage("FFmpeg path is required.") .MustAsync(async (ffmpegPath, cancellationToken) => await TestFFmpeg(ffmpegPath)) .WithMessage("FFmpeg is not installed or invalid at the specified path."); // Validate custom conversion rules RuleFor(x => x.CustomConversion) .Must(customConversions => customConversions?.All(IsValidConversionRule) != false) .WithMessage("Custom conversion rules must be in the format 'source -> target' (e.g., mp3 to flac)."); RuleFor(x => x.CustomConversion) .Must(customConversions => customConversions?.All(IsValidLossyConversion) != false) .WithMessage("Lossy formats cannot be converted to non-lossy formats."); RuleFor(x => x.CustomConversion) .Must(customConversions => customConversions?.All(IsValidCBRUsage) != false) .WithMessage("CBR flag is only applicable to lossy formats. Lossless formats are inherently variable bitrate."); RuleFor(x => x) .Must(settings => IsValidStaticConversion(settings)) .WithMessage("Lossy formats cannot be converted to non-lossy formats."); } private bool IsValidConversionRule(KeyValuePair rule) { if (string.IsNullOrWhiteSpace(rule.Key) || string.IsNullOrWhiteSpace(rule.Value)) return false; return RuleParser.TryParseRule(rule.Key, rule.Value, out _); } private bool IsValidLossyConversion(KeyValuePair rule) { if (!RuleParser.TryParseRule(rule.Key, rule.Value, out ConversionRule parsedRule)) return false; if (parsedRule.IsGlobalRule) return true; if (AudioFormatHelper.IsLossyFormat(parsedRule.SourceFormat) && !AudioFormatHelper.IsLossyFormat(parsedRule.TargetFormat)) return false; return true; } private bool IsValidCBRUsage(KeyValuePair rule) { if (!RuleParser.TryParseRule(rule.Key, rule.Value, out ConversionRule parsedRule)) return false; // If CBR is specified, target must be a lossy format if (parsedRule.UseCBR && !AudioFormatHelper.IsLossyFormat(parsedRule.TargetFormat)) return false; return true; } private static bool IsValidStaticConversion(AudioConverterSettings settings) => AudioFormatHelper.IsLossyFormat((AudioFormat)settings.TargetFormat) || (!settings.ConvertMP3 && !settings.ConvertAAC && !settings.ConvertOpus && !settings.ConvertOther); private static async Task TestFFmpeg(string ffmpegPath) { if (string.IsNullOrWhiteSpace(ffmpegPath)) return false; string oldPath = FFmpeg.ExecutablesPath; FFmpeg.SetExecutablesPath(ffmpegPath); AudioMetadataHandler.ResetFFmpegInstallationCheck(); if (!AudioMetadataHandler.CheckFFmpegInstalled()) { try { await AudioMetadataHandler.InstallFFmpeg(ffmpegPath); } catch { if (!string.IsNullOrEmpty(oldPath)) FFmpeg.SetExecutablesPath(oldPath); return false; } } return true; } } public class AudioConverterSettings : IProviderConfig { private static readonly AudioConverterSettingsValidator Validator = new(); [FieldDefinition(0, Label = "FFmpeg Path", Type = FieldType.Path, Section = MetadataSectionType.Metadata, Placeholder = "/downloads/FFmpeg", HelpText = "Specify the path to the FFmpeg binary.")] public string FFmpegPath { get; set; } = string.Empty; [FieldDefinition(1, Label = "Convert MP3", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert MP3 files.")] public bool ConvertMP3 { get; set; } [FieldDefinition(2, Label = "Convert AAC", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert AAC files.")] public bool ConvertAAC { get; set; } [FieldDefinition(3, Label = "Convert FLAC", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert FLAC files.")] public bool ConvertFLAC { get; set; } [FieldDefinition(4, Label = "Convert WAV", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert WAV files.")] public bool ConvertWAV { get; set; } [FieldDefinition(5, Label = "Convert Opus", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert Opus files.")] public bool ConvertOpus { get; set; } [FieldDefinition(7, Label = "Convert Other Formats", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Convert other formats (e.g., WMA).")] public bool ConvertOther { get; set; } [FieldDefinition(8, Label = "Target Format", Type = FieldType.Select, SelectOptions = typeof(TargetAudioFormat), Section = MetadataSectionType.Metadata, HelpText = "Select the target format to convert audio files into.")] public int TargetFormat { get; set; } = (int)TargetAudioFormat.Opus; [FieldDefinition(9, Label = "Custom Conversion Rules", Type = FieldType.KeyValueList, Section = MetadataSectionType.Metadata, HelpText = "Custom conversion rules. Examples: 'flac -> mp3:320:cbr' (FLAC to CBR MP3), 'mp3:320 -> mp3:128' (downsample), 'flac:24 -> flac:16' (reduce bit depth). Add ':cbr' for constant bitrate encoding. Upsampling is blocked automatically.")] public IEnumerable> CustomConversion { get; set; } = []; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum TargetAudioFormat { [FieldOption(Label = "AAC", Hint = "Convert to AAC format.")] AAC = 1, [FieldOption(Label = "MP3", Hint = "Convert to MP3 format.")] MP3 = 2, [FieldOption(Label = "Opus", Hint = "Convert to Opus format.")] Opus = 3, [FieldOption(Label = "FLAC", Hint = "Convert to FLAC format.")] FLAC = 5, } } ================================================ FILE: Tubifarry/Metadata/Converter/BitrateRules.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using System.Text.RegularExpressions; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Converter { public class ConversionResult { public AudioFormat TargetFormat { get; set; } public int? TargetBitrate { get; set; } public int? TargetBitDepth { get; set; } public bool UseCBR { get; set; } public bool IsBlocked { get; set; } public static ConversionResult Blocked() => new() { IsBlocked = true, TargetFormat = AudioFormat.Unknown }; public static ConversionResult Success(AudioFormat format, int? bitrate = null, int? bitDepth = null, bool useCBR = false) => new() { TargetFormat = format, TargetBitrate = bitrate, TargetBitDepth = bitDepth, UseCBR = useCBR, IsBlocked = false }; public static ConversionResult FromRule(ConversionRule rule) => new() { TargetFormat = rule.TargetFormat, TargetBitrate = rule.TargetBitrate, TargetBitDepth = rule.TargetBitDepth, UseCBR = rule.UseCBR, IsBlocked = false }; } public class ConversionRule { public AudioFormat SourceFormat { get; set; } public ComparisonOperator? SourceBitrateOperator { get; set; } public int? SourceBitrateValue { get; set; } public AudioFormat TargetFormat { get; set; } public int? TargetBitrate { get; set; } public int? TargetBitDepth { get; set; } public bool UseCBR { get; set; } public bool IsArtistRule { get; set; } // Track the type of category rule public bool IsGlobalRule { get; set; } public bool IsLossyRule { get; set; } public bool IsLosslessRule { get; set; } public bool IsCategoryRule => IsGlobalRule || IsLossyRule || IsLosslessRule; public bool MatchesBitrate(int? currentBitrate) { if (!HasBitrateConstraints()) return true; if (!currentBitrate.HasValue) return false; return EvaluateBitrateCondition(currentBitrate.Value); } public bool MatchesFormat(AudioFormat trackFormat) { if (IsGlobalRule) return true; if (IsLossyRule) return AudioFormatHelper.IsLossyFormat(trackFormat); if (IsLosslessRule) return !AudioFormatHelper.IsLossyFormat(trackFormat); return SourceFormat == trackFormat; } private bool HasBitrateConstraints() => SourceBitrateOperator.HasValue && SourceBitrateValue.HasValue; private bool EvaluateBitrateCondition(int currentBitrate) { if (!SourceBitrateOperator.HasValue || !SourceBitrateValue.HasValue) return false; return SourceBitrateOperator.Value switch { ComparisonOperator.Equal => currentBitrate == SourceBitrateValue.Value, ComparisonOperator.NotEqual => currentBitrate != SourceBitrateValue.Value, ComparisonOperator.LessThan => currentBitrate < SourceBitrateValue.Value, ComparisonOperator.LessThanOrEqual => currentBitrate <= SourceBitrateValue.Value, ComparisonOperator.GreaterThan => currentBitrate > SourceBitrateValue.Value, ComparisonOperator.GreaterThanOrEqual => currentBitrate >= SourceBitrateValue.Value, _ => false }; } private string GetOperatorSymbol() => SourceBitrateOperator.HasValue ? OperatorSymbols.GetSymbol(SourceBitrateOperator.Value) : string.Empty; public override string ToString() => $"{FormatSourcePart()}->{FormatTargetPart()}"; private string FormatSourcePart() { string source; if (IsGlobalRule) source = RuleParser.GlobalRuleIdentifier; else if (IsLossyRule) source = RuleParser.LossyRuleIdentifier; else if (IsLosslessRule) source = RuleParser.LosslessRuleIdentifier; else source = SourceFormat.ToString(); if (HasBitrateConstraints()) source += GetOperatorSymbol() + SourceBitrateValue!.Value; return source; } private string FormatTargetPart() { string target = TargetFormat.ToString(); if (TargetBitrate.HasValue) { target += ":" + TargetBitrate.Value.ToString(); if (UseCBR) target += ":cbr"; } else if (TargetBitDepth.HasValue) target += ":" + TargetBitDepth.Value.ToString(); return target; } } public static class OperatorSymbols { public const string Equal = "="; public const string NotEqual = "!="; public const string LessThan = "<"; public const string LessThanOrEqual = "<="; public const string GreaterThan = ">"; public const string GreaterThanOrEqual = ">="; public static string GetSymbol(ComparisonOperator op) { return op switch { ComparisonOperator.Equal => Equal, ComparisonOperator.NotEqual => NotEqual, ComparisonOperator.LessThan => LessThan, ComparisonOperator.LessThanOrEqual => LessThanOrEqual, ComparisonOperator.GreaterThan => GreaterThan, ComparisonOperator.GreaterThanOrEqual => GreaterThanOrEqual, _ => string.Empty }; } public static ComparisonOperator? FromSymbol(string symbol) { return symbol switch { Equal => ComparisonOperator.Equal, NotEqual => ComparisonOperator.NotEqual, LessThan => ComparisonOperator.LessThan, LessThanOrEqual => ComparisonOperator.LessThanOrEqual, GreaterThan => ComparisonOperator.GreaterThan, GreaterThanOrEqual => ComparisonOperator.GreaterThanOrEqual, _ => null }; } } public enum ComparisonOperator { Equal, NotEqual, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual } public static partial class RuleParser { public const string GlobalRuleIdentifier = "all"; public const string LossyRuleIdentifier = "lossy"; public const string LosslessRuleIdentifier = "lossless"; public const string NoConversionTag = "no-conversion"; private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(RuleParser)); public static bool TryParseRule(string sourceKey, string targetValue, out ConversionRule rule) { _logger.Debug("Parsing rule: {0} -> {1}", sourceKey, targetValue); rule = new ConversionRule(); if (string.IsNullOrWhiteSpace(sourceKey) || string.IsNullOrWhiteSpace(targetValue)) { _logger.Debug("Rule parsing failed: Empty source or target"); return false; } return ParseSourcePart(sourceKey.Trim(), rule) && ParseTargetPart(targetValue.Trim(), rule); } public static bool TryParseArtistTag(string tagLabel, out ConversionRule rule) { _logger.Debug("Parsing artist tag: {0}", tagLabel); rule = new ConversionRule { IsArtistRule = true }; if (string.IsNullOrWhiteSpace(tagLabel)) return false; // Handle no-conversion tag if (string.Equals(tagLabel, NoConversionTag, StringComparison.OrdinalIgnoreCase)) { rule.TargetFormat = AudioFormat.Unknown; return true; } // Match format like "opus" or format+bitrate like "opus192" Match match = ArtistTagRegex().Match(tagLabel.Trim()); if (!match.Success) { _logger.Debug("Invalid artist tag format: {0}", tagLabel); return false; } string formatName = match.Groups[1].Value; if (!Enum.TryParse(formatName, true, out AudioFormat targetFormat)) { _logger.Debug("Invalid format in artist tag: {0}", formatName); return false; } rule.TargetFormat = targetFormat; // Parse bitrate if present if (match.Groups[2].Success && int.TryParse(match.Groups[2].Value, out int bitrate)) { int clampedBitrate = AudioFormatHelper.ClampBitrate(targetFormat, bitrate); rule.TargetBitrate = AudioFormatHelper.RoundToStandardBitrate(clampedBitrate); } return true; } private static bool ParseSourcePart(string sourceKey, ConversionRule rule) { Match sourceMatch = SourceFormatRegex().Match(sourceKey); if (!sourceMatch.Success) { _logger.Debug("Invalid source format pattern: {0}", sourceKey); return false; } if (!ParseSourceFormat(sourceMatch.Groups[1].Value, rule)) return false; if (sourceMatch.Groups[2].Success && sourceMatch.Groups[3].Success) { // Category rules (all, lossy, lossless) cannot have bitrate constraints if (rule.IsCategoryRule) { _logger.Warn("Invalid: Bitrate constraints not applicable to category rules (all, lossy, lossless)"); return false; } if (!AudioFormatHelper.IsLossyFormat(rule.SourceFormat)) { _logger.Warn("Invalid: Bitrate constraints not applicable to lossless format"); return false; } if (!ParseSourceBitrateConstraints(sourceMatch.Groups[2].Value, sourceMatch.Groups[3].Value, rule)) return false; } return true; } private static bool ParseSourceFormat(string formatName, ConversionRule rule) { if (string.Equals(formatName, GlobalRuleIdentifier, StringComparison.OrdinalIgnoreCase)) { rule.SourceFormat = AudioFormat.Unknown; rule.IsGlobalRule = true; return true; } if (string.Equals(formatName, LossyRuleIdentifier, StringComparison.OrdinalIgnoreCase)) { rule.SourceFormat = AudioFormat.Unknown; rule.IsLossyRule = true; return true; } if (string.Equals(formatName, LosslessRuleIdentifier, StringComparison.OrdinalIgnoreCase)) { rule.SourceFormat = AudioFormat.Unknown; rule.IsLosslessRule = true; return true; } if (!Enum.TryParse(formatName, true, out AudioFormat sourceFormat)) { _logger.Debug("Invalid source format: {0}", formatName); return false; } rule.SourceFormat = sourceFormat; return true; } private static bool ParseSourceBitrateConstraints(string operatorStr, string bitrateStr, ConversionRule rule) { if (!int.TryParse(bitrateStr, out int bitrateValue)) { _logger.Debug("Invalid source bitrate value: {0}", bitrateStr); return false; } ComparisonOperator? comparisonOp = OperatorSymbols.FromSymbol(operatorStr); if (!comparisonOp.HasValue) { _logger.Debug("Invalid comparison operator: {0}", operatorStr); return false; } rule.SourceBitrateOperator = comparisonOp.Value; rule.SourceBitrateValue = bitrateValue; return true; } private static bool ParseTargetPart(string targetValue, ConversionRule rule) { Match targetMatch = TargetFormatRegex().Match(targetValue); if (!targetMatch.Success) { _logger.Debug("Invalid target format pattern: {0}", targetValue); return false; } if (!ParseTargetFormat(targetMatch.Groups[1].Value, rule)) return false; if (targetMatch.Groups[2].Success && !ParseTargetBitrate(targetMatch.Groups[2].Value, rule)) return false; // Check for CBR flag (Group 3) if (targetMatch.Groups[3].Success && targetMatch.Groups[3].Value == "cbr") { // CBR only makes sense for lossy formats if (!AudioFormatHelper.IsLossyFormat(rule.TargetFormat)) { _logger.Warn("CBR flag is not applicable to lossless format {0}. Lossless formats are inherently variable bitrate.", rule.TargetFormat); return false; } rule.UseCBR = true; } return true; } private static bool ParseTargetFormat(string formatName, ConversionRule rule) { if (!Enum.TryParse(formatName, true, out AudioFormat targetFormat)) { _logger.Debug("Invalid target format: {0}", formatName); return false; } if (!AudioMetadataHandler.IsTargetFormatSupportedForEncoding(targetFormat)) { _logger.Warn("Target format {0} is not supported for encoding by FFmpeg", targetFormat); return false; } rule.TargetFormat = targetFormat; return true; } private static bool ParseTargetBitrate(string bitrateStr, ConversionRule rule) { if (!int.TryParse(bitrateStr, out int targetValue)) { _logger.Debug("Invalid target value: {0}", bitrateStr); return false; } if (!AudioFormatHelper.IsLossyFormat(rule.TargetFormat)) { if (targetValue != 16 && targetValue != 24 && targetValue != 32) { _logger.Warn("Invalid bit depth {0} for lossless format {1}. Must be 16, 24, or 32", targetValue, rule.TargetFormat); return false; } rule.TargetBitDepth = targetValue; return true; } int clampedBitrate = AudioFormatHelper.ClampBitrate(rule.TargetFormat, targetValue); if (clampedBitrate != targetValue) { _logger.Debug("Target bitrate ({0}) outside of valid range for format {1}", targetValue, rule.TargetFormat); return false; } rule.TargetBitrate = AudioFormatHelper.RoundToStandardBitrate(targetValue); return true; } [GeneratedRegex(@"^([a-zA-Z0-9]+)(?:([!<>=]{1,2})(\d+))?$", RegexOptions.Compiled)] private static partial Regex SourceFormatRegex(); [GeneratedRegex(@"^([a-zA-Z0-9]+)(?::(\d+)k?)?(?::(cbr))?$", RegexOptions.Compiled)] private static partial Regex TargetFormatRegex(); [GeneratedRegex(@"^([a-zA-Z]+)(?:-(\d+)k?)?$", RegexOptions.Compiled)] private static partial Regex ArtistTagRegex(); } } ================================================ FILE: Tubifarry/Metadata/Lyrics/LyricEnhancerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Metadata.Lyrics { public class LyricsEnhancerSettingsValidator : AbstractValidator { public LyricsEnhancerSettingsValidator() { // Validate LRCLIB instance URL if enabled RuleFor(x => x.LrcLibInstanceUrl) .NotEmpty() .When(x => x.LrcLibEnabled) .WithMessage("LRCLIB instance URL is required when LRCLIB provider is enabled") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .When(x => x.LrcLibEnabled && !string.IsNullOrEmpty(x.LrcLibInstanceUrl)) .WithMessage("LRCLIB instance URL must be a valid URL"); // Validate Genius API key if enabled RuleFor(x => x.GeniusApiKey) .NotEmpty() .When(x => x.GeniusEnabled) .WithMessage("Genius API key is required when Genius provider is enabled"); // Validate at least one provider is enabled RuleFor(x => new { x.LrcLibEnabled, x.GeniusEnabled }) .Must(x => x.LrcLibEnabled || x.GeniusEnabled) .WithMessage("At least one lyrics provider must be enabled"); // Validate UpdateInterval when scheduled updates are enabled RuleFor(x => x.UpdateInterval) .GreaterThanOrEqualTo(7) .When(x => x.EnableScheduledUpdates) .WithMessage("Update interval must be at least 1 week"); } } public class LyricsEnhancerSettings : IProviderConfig { private static readonly LyricsEnhancerSettingsValidator Validator = new(); [FieldDefinition(0, Label = "Create LRC Files", Type = FieldType.Select, SelectOptions = typeof(LyricOptions), Section = MetadataSectionType.Metadata, HelpText = "Choose what kind of LRC files to create")] public int LrcFileOptions { get; set; } = (int)LyricOptions.OnlySynced; [FieldDefinition(1, Label = "Lyrics Embedding", Type = FieldType.Select, SelectOptions = typeof(LyricOptions), Section = MetadataSectionType.Metadata, HelpText = "Choose how to embed lyrics in audio files metadata")] public int LyricEmbeddingOption { get; set; } = (int)LyricOptions.Disabled; [FieldDefinition(2, Label = "Overwrite Existing LRC Files", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Overwrite existing LRC files")] public bool OverwriteExistingLrcFiles { get; set; } // LRCLIB Provider settings [FieldDefinition(3, Label = "Enable LRCLIB", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Use LRCLIB as a lyrics provider (provides synced lyrics)")] public bool LrcLibEnabled { get; set; } [FieldDefinition(4, Label = "LRCLIB Instance URL", Type = FieldType.Url, Section = MetadataSectionType.Metadata, HelpText = "URL of the LRCLIB instance to use", Placeholder = "https://lrclib.net")] public string LrcLibInstanceUrl { get; set; } = "https://lrclib.net"; // Genius Provider settings [FieldDefinition(5, Label = "Enable Genius", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Use Genius as a lyrics provider (text only, no synced lyrics)")] public bool GeniusEnabled { get; set; } [FieldDefinition(6, Label = "Genius API Key", Type = FieldType.Textbox, Section = MetadataSectionType.Metadata, HelpText = "Your Genius API key", Privacy = PrivacyLevel.ApiKey)] public string GeniusApiKey { get; set; } = ""; // Scheduled Update Settings [FieldDefinition(7, Label = "Enable Scheduled Updates", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Enable automatic scheduled updates to refresh lyrics for existing files")] public bool EnableScheduledUpdates { get; set; } [FieldDefinition(8, Label = "Update Interval", Type = FieldType.Number, Unit = "days", Section = MetadataSectionType.Metadata, HelpText = "How often to run scheduled lyrics updates.")] public int UpdateInterval { get; set; } = 7; public LyricsEnhancerSettings() => Instance = this; public static LyricsEnhancerSettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } /// /// Command for scheduled lyrics update task. /// public class LyricsUpdateCommand : Command { public override bool SendUpdatesToClient => true; public override bool UpdateScheduledTask => true; public override string CompletionMessage => _completionMessage ?? "Lyrics update completed"; private string? _completionMessage; public void SetCompletionMessage(string message) => _completionMessage = message; } public enum LyricOptions { [FieldOption(Label = "Disabled", Hint = "Disabled")] Disabled, [FieldOption(Label = "Only Plain", Hint = "Use plain text lyrics if available.")] OnlyPlain, [FieldOption(Label = "Only Synced", Hint = "Use synced lyrics if available.")] OnlySynced, [FieldOption(Label = "Prefer Synced", Hint = "Use synced lyrics if available, fall back to plain text.")] PrefferSynced } } ================================================ FILE: Tubifarry/Metadata/Lyrics/LyricsEnhancer.cs ================================================ using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.Datastore; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.Extras.Lyrics; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Extras.Metadata.Files; using NzbDrone.Core.Extras.Others; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using Tubifarry.Core.Records; using Tubifarry.Metadata.ScheduledTasks; namespace Tubifarry.Metadata.Lyrics { public class LyricsEnhancer : ScheduledTaskBase, IExecute { private readonly Logger _logger; private readonly HttpClient _httpClient; private readonly IRootFolderWatchingService _rootFolderWatchingService; private readonly IArtistService _artistService; private readonly IDiskProvider _diskProvider; private readonly TrackFileRepositoryHelper _trackFileRepositoryHelper; private readonly IMediaFileService _mediaFileService; private LyricsProviders _lyricsProviders; // Batch size for SQL LIMIT/OFFSET pagination private const int SQL_BATCH_SIZE = 500; public LyricsEnhancer( HttpClient httpClient, Logger logger, IRootFolderWatchingService rootFolderWatchingService, ILyricFileService lyricFileService, IArtistService artistService, IDiskProvider diskProvider, IMainDatabase database, IEventAggregator eventAggregator, ITrackRepository trackRepository, IExtraFileService otherExtraFileService, IMediaFileService mediaFileService) { _logger = logger; _httpClient = httpClient; _rootFolderWatchingService = rootFolderWatchingService; _lyricsProviders = new LyricsProviders(httpClient, logger, ActiveSettings); _artistService = artistService; _diskProvider = diskProvider; _mediaFileService = mediaFileService; _trackFileRepositoryHelper = new TrackFileRepositoryHelper(database, eventAggregator, trackRepository, lyricFileService, otherExtraFileService, logger); } public override string Name => "Lyrics Enhancer"; public override Type CommandType => typeof(LyricsUpdateCommand); public override int IntervalMinutes => ActiveSettings.EnableScheduledUpdates ? (int)TimeSpan.FromDays(ActiveSettings.UpdateInterval).TotalMinutes : 0; private LyricsEnhancerSettings ActiveSettings => Settings ?? LyricsEnhancerSettings.Instance!; public override CommandPriority Priority => CommandPriority.Low; public void Execute(LyricsUpdateCommand message) { if (!ActiveSettings.EnableScheduledUpdates) { _logger.Debug("Scheduled lyrics updates are disabled in settings"); message.SetCompletionMessage("Lyrics updates are disabled"); return; } try { _lyricsProviders = new LyricsProviders(_httpClient, _logger, ActiveSettings); _logger.ProgressInfo("Starting scheduled lyrics update"); int totalTracks = _trackFileRepositoryHelper.GetTracksWithoutLrcFilesCount(); if (totalTracks == 0) { _logger.Info("All tracks in database have lyric file entries"); message.SetCompletionMessage("All tracks have lyrics entries"); return; } _logger.Debug($"Found {totalTracks} tracks without lyric entries in database"); ProcessingResult totalResult = new(); int processedCount = 0; for (int offset = 0; offset < totalTracks; offset += SQL_BATCH_SIZE) { List batch = _trackFileRepositoryHelper.GetTracksWithoutLrcFilesBatch(offset, SQL_BATCH_SIZE); if (batch.Count == 0) break; _logger.Debug($"Processing SQL batch {(offset / SQL_BATCH_SIZE) + 1} (offset {offset}, {batch.Count} tracks)"); ProcessingResult batchResult = ProcessTrackBatch(batch); totalResult.SuccessCount += batchResult.SuccessCount; totalResult.SyncedCount += batchResult.SyncedCount; totalResult.FailedCount += batchResult.FailedCount; processedCount += batch.Count; _logger.Debug($"Progress: {processedCount}/{totalTracks} tracks without lyric processed"); } string completionMsg = $"Lyrics update completed: {totalResult.SuccessCount} created, " + $"{totalResult.SyncedCount} synced, " + $"{totalResult.FailedCount} not found."; _logger.Info(completionMsg); message.SetCompletionMessage(completionMsg); } catch (Exception ex) { _logger.Error(ex, "Error during scheduled lyrics update execution"); message.SetCompletionMessage($"Lyrics update failed: {ex.Message}"); } } private ProcessingResult ProcessTrackBatch(List batch) { ProcessingResult result = new(); foreach (TrackFile trackFile in batch) { try { Artist? artist = trackFile.Artist?.Value ?? _artistService.GetArtist(trackFile.Tracks?.Value?.FirstOrDefault()?.Artist?.Value?.Id ?? 0); if (artist == null) { _logger.Debug($"Could not find artist for track file: {trackFile.Path}"); result.FailedCount++; continue; } if (LyricsHelper.LrcFileExistsOnDisk(trackFile.Path, _diskProvider)) { SyncExistingLrcFile(artist, trackFile); result.SyncedCount++; continue; } _logger.ProgressTrace($"Searching lyrics for: {trackFile.Tracks?.Value?.FirstOrDefault()?.Title ?? Path.GetFileName(trackFile.Path)}"); MetadataFileResult? metadataResult = TrackMetadata(artist, trackFile); if (metadataResult != null && !string.IsNullOrEmpty(metadataResult.Contents)) { WriteLrcFile(artist, trackFile, metadataResult); result.SuccessCount++; } else { result.FailedCount++; _logger.Trace($"No lyrics found for: {trackFile.Path}"); } } catch (Exception ex) { _logger.Error(ex, $"Error processing track: {trackFile.Path}"); result.FailedCount++; } } return result; } private void SyncExistingLrcFile(Artist artist, TrackFile trackFile) { string lrcPath = Path.ChangeExtension(trackFile.Path, ".lrc"); string relativePath = artist.Path.GetRelativePath(lrcPath); _trackFileRepositoryHelper.CreateAndUpsertLyricFile(artist, trackFile, relativePath); _logger.Trace($"Synced existing LRC file to database: {lrcPath}"); } private void WriteLrcFile(Artist artist, TrackFile trackFile, MetadataFileResult metadataResult) { string lrcPath = Path.Combine(artist.Path, metadataResult.RelativePath); _diskProvider.WriteAllText(lrcPath, metadataResult.Contents); _trackFileRepositoryHelper.CreateAndUpsertLyricFile(artist, trackFile, metadataResult.RelativePath); _logger.Trace($"Created LRC file: {lrcPath}"); } private void EmbedLyrics(Lyric lyric, TrackFile trackFile) { LyricOptions embeddingOption = (LyricOptions)ActiveSettings.LyricEmbeddingOption; if (embeddingOption == LyricOptions.Disabled) return; string? lyricsToEmbed = LyricsHelper.GetLyricsForEmbedding(lyric, embeddingOption); if (!string.IsNullOrWhiteSpace(lyricsToEmbed)) { bool wasModified = LyricsHelper.EmbedLyricsInAudioFile(trackFile.Path, lyricsToEmbed, _logger, _rootFolderWatchingService); if (wasModified && trackFile.Id > 0) { try { FileInfo fileInfo = new(trackFile.Path); trackFile.Size = fileInfo.Length; trackFile.Modified = fileInfo.LastWriteTimeUtc; _mediaFileService.Update(trackFile); _logger.Debug($"Updated TrackFile metadata after embedding lyrics: Size={trackFile.Size}, Modified={trackFile.Modified:O}"); } catch (Exception ex) { _logger.Warn(ex, $"Failed to update TrackFile metadata after embedding lyrics: {trackFile.Path}"); } } } } public override string GetFilenameAfterMove(Artist artist, TrackFile trackFile, MetadataFile metadataFile) { if (metadataFile.Type == MetadataType.TrackMetadata) return Path.ChangeExtension(trackFile.Path, ".lrc"); _logger.Trace("Unknown track file metadata: {0}", metadataFile.RelativePath); return Path.Combine(artist.Path, metadataFile.RelativePath); } public override MetadataFileResult TrackMetadata(Artist artist, TrackFile trackFile) { if (!ActiveSettings.OverwriteExistingLrcFiles && LyricsHelper.LrcFileExistsOnDisk(trackFile.Path, _diskProvider)) { _logger.Trace($"LRC file already exists and overwrite is disabled: {trackFile.Path}"); return default!; } if (!_diskProvider.FileExists(trackFile.Path)) { _logger.Warn($"Track file does not exist: {trackFile.Path}"); return default!; } try { return ProcessTrackLyricsAsync(artist, trackFile).GetAwaiter().GetResult(); } catch (Exception ex) { _logger.Error(ex, $"Error processing lyrics for track: {trackFile.Path}"); return default!; } } private async Task ProcessTrackLyricsAsync(Artist artist, TrackFile trackFile) { _logger.Trace($"Processing lyrics for track: {trackFile.Path}"); (string Artist, string Title, string Album, int Duration)? trackInfo = LyricsHelper.ExtractTrackInfo(trackFile, artist, _logger); if (trackInfo == null) return default!; Lyric? lyric = await FetchLyricsAsync(trackInfo.Value); if (lyric == null) { _logger.Trace($"No lyrics found for track: {trackInfo.Value.Title} by {trackInfo.Value.Artist}"); return default!; } // Embed lyrics into audio file based on settings EmbedLyrics(lyric, trackFile); // Create LRC file content based on settings string? lrcContent = CreateLrcFileContent(lyric, trackInfo.Value); if (string.IsNullOrEmpty(lrcContent)) return default!; string relativePath = artist.Path.GetRelativePath(trackFile.Path); relativePath = Path.ChangeExtension(relativePath, ".lrc"); return new MetadataFileResult(relativePath, lrcContent); } private async Task FetchLyricsAsync((string Artist, string Title, string Album, int Duration) trackInfo) { Lyric? lyric = null; if (ActiveSettings.LrcLibEnabled) { lyric = await _lyricsProviders.FetchFromLrcLibAsync( trackInfo.Artist, trackInfo.Title, trackInfo.Album, trackInfo.Duration); } if (lyric == null && ActiveSettings.GeniusEnabled && !string.IsNullOrWhiteSpace(ActiveSettings.GeniusApiKey)) { lyric = await _lyricsProviders.FetchFromGeniusAsync(trackInfo.Artist, trackInfo.Title); } return lyric; } private string? CreateLrcFileContent(Lyric lyric, (string Artist, string Title, string Album, int Duration) trackInfo) { LyricOptions lrcOption = (LyricOptions)ActiveSettings.LrcFileOptions; return LyricsHelper.GetLyricsForLrcFile( lyric, lrcOption, trackInfo.Artist, trackInfo.Title, trackInfo.Album, trackInfo.Duration); } /// /// Result container for track processing operations. /// private class ProcessingResult { public int SuccessCount { get; set; } public int SyncedCount { get; set; } public int FailedCount { get; set; } } } } ================================================ FILE: Tubifarry/Metadata/Lyrics/LyricsHelper.cs ================================================ using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Music; using System.Text; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Lyrics { /// /// Helper methods for lyrics processing, matching, file operations, and track info extraction. /// public static class LyricsHelper { public static JToken? ScoreAndSelectBestMatch(List artistMatches, List songHits, string artistName, string trackTitle, Logger logger) { JToken? bestMatch = null; int bestScore = 0; List candidatesToScore = artistMatches.Count > 0 ? artistMatches : songHits; logger.Trace("Beginning enhanced fuzzy matching process..."); foreach (JToken hit in candidatesToScore) { string resultTitle = hit["result"]?["title"]?.ToString() ?? string.Empty; string resultArtist = hit["result"]?["primary_artist"]?["name"]?.ToString() ?? string.Empty; int tokenSetScore = FuzzySharp.Fuzz.TokenSetRatio(resultTitle, trackTitle); int tokenSortScore = FuzzySharp.Fuzz.TokenSortRatio(resultTitle, trackTitle); int partialRatio = FuzzySharp.Fuzz.PartialRatio(resultTitle, trackTitle); int weightedRatio = FuzzySharp.Fuzz.WeightedRatio(resultTitle, trackTitle); int titleScore = Math.Max(Math.Max(tokenSetScore, tokenSortScore), Math.Max(partialRatio, weightedRatio)); int artistScore = artistMatches.Count > 0 ? 100 : FuzzySharp.Fuzz.WeightedRatio(resultArtist, artistName); int combinedScore = ((titleScore * 3) + (artistScore * 7)) / 10; logger.Debug($"Match candidate: '{resultTitle}' by '{resultArtist}' - " + $"Title Score: {titleScore} (Token Set: {tokenSetScore}, Token Sort: {tokenSortScore}, " + $"Partial: {partialRatio}, Weighted: {weightedRatio}), " + $"Artist Score: {artistScore}, Combined: {combinedScore}"); if (combinedScore > bestScore) { bestScore = combinedScore; bestMatch = hit; logger.Debug($"New best match found with score: {combinedScore}"); } } if (bestMatch == null || bestScore < 70) { logger.Warn($"Match score below threshold (70%). No lyrics will be selected for: '{trackTitle}' by '{artistName}'"); return null; } return bestMatch; } public static string? CreateRawLrcContent(List? syncedLyrics) { if (syncedLyrics == null || syncedLyrics.Count == 0) return null; IEnumerable lines = syncedLyrics .Where(l => l != null && !string.IsNullOrEmpty(l.LrcTimestamp) && !string.IsNullOrEmpty(l.Line)) .OrderBy(l => double.TryParse(l.Milliseconds ?? "0", out double ms) ? ms : 0) .Select(l => $"{l.LrcTimestamp} {l.Line}"); return string.Join(Environment.NewLine, lines); } public static string? CreateLrcFileContent(Lyric lyric, string artistName, string trackTitle, string albumName, int duration) { string? rawLrc = CreateRawLrcContent(lyric.SyncedLyrics); if (rawLrc == null) return null; StringBuilder lrcContent = new(); lrcContent.AppendLine($"[ar:{artistName}]"); if (!string.IsNullOrEmpty(albumName)) lrcContent.AppendLine($"[al:{albumName}]"); lrcContent.AppendLine($"[ti:{trackTitle}]"); if (duration > 0) { TimeSpan ts = TimeSpan.FromSeconds(duration); lrcContent.AppendLine($"[length:{ts:mm\\:ss\\.ff}]"); } lrcContent.AppendLine("[by:Tubifarry Lyrics Enhancer]"); lrcContent.AppendLine(); lrcContent.Append(rawLrc); return lrcContent.ToString(); } public static string? GetLyricsForEmbedding(Lyric lyric, LyricOptions option) => option switch { LyricOptions.Disabled => null, LyricOptions.OnlyPlain => lyric.PlainLyrics, LyricOptions.OnlySynced => CreateRawLrcContent(lyric.SyncedLyrics), LyricOptions.PrefferSynced => CreateRawLrcContent(lyric.SyncedLyrics) ?? lyric.PlainLyrics, _ => null }; public static string? GetLyricsForLrcFile(Lyric lyric, LyricOptions option, string artistName, string trackTitle, string albumName, int duration) => option switch { LyricOptions.Disabled => null, LyricOptions.OnlyPlain => lyric.PlainLyrics, LyricOptions.OnlySynced => CreateLrcFileContent(lyric, artistName, trackTitle, albumName, duration), LyricOptions.PrefferSynced => CreateLrcFileContent(lyric, artistName, trackTitle, albumName, duration) ?? lyric.PlainLyrics, _ => null }; public static bool EmbedLyricsInAudioFile(string filePath, string lyrics, Logger logger, IRootFolderWatchingService rootFolderWatchingService) { try { rootFolderWatchingService.ReportFileSystemChangeBeginning(filePath); using (TagLib.File file = TagLib.File.Create(filePath)) { file.Tag.Lyrics = lyrics; file.Save(); } logger.Trace($"Embedded lyrics in file: {filePath}"); return true; } catch (Exception ex) { logger.Error(ex, $"Failed to embed lyrics in file: {filePath}"); return false; } } public static (string Artist, string Title, string Album, int Duration)? ExtractTrackInfo(TrackFile trackFile, Artist artist, Logger logger) { if (trackFile.Tracks?.Value == null || trackFile.Tracks.Value.Count == 0) { logger.Warn($"No tracks found for file: {trackFile.Path}"); return null; } Track? track = trackFile.Tracks.Value.FirstOrDefault(x => x != null); if (track == null) { logger.Warn($"No track information found for file: {trackFile.Path}"); return null; } Album? album = track.Album; string trackTitle = track.Title; string artistName = artist.Name; string albumName = album?.Title ?? track?.AlbumRelease?.Value?.Album?.Value?.Title ?? trackFile.Tracks.Value.FirstOrDefault(x => !string.IsNullOrEmpty(x?.Album?.Title))?.Album?.Title ?? ""; int trackDuration = 0; if (track!.Duration > 0) trackDuration = (int)Math.Round(TimeSpan.FromMilliseconds(track.Duration).TotalSeconds); return (artistName, trackTitle, albumName, trackDuration); } public static bool LrcFileExistsOnDisk(string trackFilePath, IDiskProvider diskProvider) { if (string.IsNullOrEmpty(trackFilePath)) return false; string lrcPath = Path.ChangeExtension(trackFilePath, ".lrc"); return diskProvider.FileExists(lrcPath); } } } ================================================ FILE: Tubifarry/Metadata/Lyrics/LyricsProviders.cs ================================================ using Newtonsoft.Json.Linq; using NLog; using System.Text.RegularExpressions; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Lyrics { /// /// Handles fetching lyrics from LRCLIB and Genius APIs. /// public partial class LyricsProviders { private readonly HttpClient _httpClient; private readonly Logger _logger; private readonly LyricsEnhancerSettings _settings; public LyricsProviders(HttpClient httpClient, Logger logger, LyricsEnhancerSettings settings) { _httpClient = httpClient; _logger = logger; _settings = settings; } #region LRCLIB Provider public async Task FetchFromLrcLibAsync(string artistName, string trackTitle, string albumName, int duration) { try { string requestUri = $"{_settings.LrcLibInstanceUrl}/api/get?artist_name={Uri.EscapeDataString(artistName)}&track_name={Uri.EscapeDataString(trackTitle)}{(string.IsNullOrEmpty(albumName) ? "" : $"&album_name={Uri.EscapeDataString(albumName)}")}{(duration != 0 ? $"&duration={duration}" : "")}"; _logger.Trace($"Requesting lyrics from LRCLIB: {requestUri}"); HttpResponseMessage response = await _httpClient.GetAsync(requestUri); if (!response.IsSuccessStatusCode) { if (response.StatusCode == System.Net.HttpStatusCode.NotFound) _logger.Debug($"No lyrics found on LRCLIB for track: {trackTitle} by {artistName}"); else _logger.Debug($"Failed to fetch lyrics from LRCLIB. Status: {response.StatusCode}"); return null; } string content = await response.Content.ReadAsStringAsync(); JObject? json = JObject.Parse(content); if (json == null) { _logger.Warn("Failed to parse JSON response from LRCLIB"); return null; } string plainLyrics = json["plainLyrics"]?.ToString() ?? string.Empty; string syncedLyricsStr = json["syncedLyrics"]?.ToString() ?? string.Empty; List? syncedLyrics = SyncLine.ParseSyncedLyrics(syncedLyricsStr); if (string.IsNullOrWhiteSpace(plainLyrics) && (syncedLyrics == null || syncedLyrics.Count == 0)) { _logger.Debug($"No lyrics found from LRCLIB for track: {trackTitle} by {artistName}"); return null; } return new Lyric(plainLyrics, syncedLyrics); } catch (Exception ex) { _logger.Error(ex, $"Error fetching lyrics from LRCLIB for track: {trackTitle} by {artistName}"); return null; } } #endregion LRCLIB Provider #region Genius Provider public async Task FetchFromGeniusAsync(string artistName, string trackTitle) { try { JToken? bestMatch = await SearchSongOnGeniusAsync(artistName, trackTitle); if (bestMatch == null) return null; string? songPath = bestMatch["result"]?["path"]?.ToString(); if (string.IsNullOrEmpty(songPath)) { _logger.Warn("Could not find song path in Genius response"); return null; } string? plainLyrics = await ExtractLyricsFromGeniusPageAsync(songPath); if (string.IsNullOrWhiteSpace(plainLyrics)) return null; return new Lyric(plainLyrics, null); } catch (Exception ex) { _logger.Error(ex, $"Error fetching lyrics from Genius for track: {trackTitle} by {artistName}"); return null; } } private async Task SearchSongOnGeniusAsync(string artistName, string trackTitle) { string searchUrl = $"https://api.genius.com/search?q={Uri.EscapeDataString($"{artistName} {trackTitle}")}"; _logger.Debug($"Searching for track on Genius: {searchUrl}"); using HttpRequestMessage request = new(HttpMethod.Get, searchUrl); request.Headers.Add("Authorization", $"Bearer {_settings.GeniusApiKey}"); HttpResponseMessage response = await _httpClient.SendAsync(request); if (!response.IsSuccessStatusCode) { _logger.Warn($"Failed to search Genius. Status: {response.StatusCode}"); return null; } string responseContent = await response.Content.ReadAsStringAsync(); JObject? searchJson = JObject.Parse(responseContent); if (searchJson?["response"] == null) { _logger.Warn("Invalid response format from Genius API"); return null; } if (searchJson["response"]?["hits"] is not JArray hits || hits.Count == 0) { _logger.Debug($"No results found on Genius for: {trackTitle} by {artistName}"); return null; } List songHits = hits.Where(h => h["type"]?.ToString() == "song" && h["result"] != null).ToList(); if (songHits.Count == 0) { _logger.Debug("No songs found in search results"); return null; } List artistMatches = songHits.Where(h => string.Equals(h["result"]?["primary_artist"]?["name"]?.ToString() ?? string.Empty, artistName, StringComparison.OrdinalIgnoreCase)).ToList(); _logger.Trace($"Found {artistMatches.Count} tracks by exact artist name '{artistName}'"); return LyricsHelper.ScoreAndSelectBestMatch(artistMatches, songHits, artistName, trackTitle, _logger); } private async Task ExtractLyricsFromGeniusPageAsync(string songPath) { string songUrl = $"https://genius.com{songPath}"; _logger.Trace($"Fetching lyrics from Genius page: {songUrl}"); HttpResponseMessage? pageResponse = await _httpClient.GetAsync(songUrl); if (pageResponse?.IsSuccessStatusCode != true) { _logger.Warn($"Failed to fetch Genius lyrics page. Status: {pageResponse?.StatusCode}"); return null; } string html = await pageResponse.Content.ReadAsStringAsync(); _logger.Trace("Attempting to extract lyrics using multiple regex patterns"); string? plainLyrics = ExtractLyricsFromHtml(html); if (string.IsNullOrWhiteSpace(plainLyrics)) { _logger.Debug("Extracted lyrics from Genius are empty"); return null; } return plainLyrics; } private string? ExtractLyricsFromHtml(string html) { Match match = DataLyricsContainerRegex().Match(html); if (!match.Success) match = ClassicLyricsClassRegex().Match(html); if (!match.Success) match = LyricsRootIdRegex().Match(html); if (match.Success) { _logger.Trace("Match found. Processing lyrics HTML..."); string lyricsHtml = match.Groups[1].Value; string plainLyrics = BrTagRegex().Replace(lyricsHtml, "\n"); plainLyrics = ItalicTagRegex().Replace(plainLyrics, ""); plainLyrics = BoldTagRegex().Replace(plainLyrics, ""); plainLyrics = AnchorTagRegex().Replace(plainLyrics, ""); plainLyrics = AllHtmlTagsRegex().Replace(plainLyrics, ""); plainLyrics = System.Web.HttpUtility.HtmlDecode(plainLyrics).Trim(); return plainLyrics; } else { _logger.Debug("No matching lyrics pattern found in HTML"); return null; } } [GeneratedRegex(@"]*data-lyrics-container[^>]*>(.*?)<\/div>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline, "de-DE")] private static partial Regex DataLyricsContainerRegex(); [GeneratedRegex(@"]*class=""[^""]*lyrics[^""]*""[^>]*>(.*?)<\/div>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline, "de-DE")] private static partial Regex ClassicLyricsClassRegex(); [GeneratedRegex(@"]*id=""lyrics-root[^""]*""[^>]*>(.*?)<\/div>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline, "de-DE")] private static partial Regex LyricsRootIdRegex(); [GeneratedRegex(@"]*>", RegexOptions.Compiled)] private static partial Regex BrTagRegex(); [GeneratedRegex(@"]*>", RegexOptions.Compiled)] private static partial Regex ItalicTagRegex(); [GeneratedRegex(@"]*>", RegexOptions.Compiled)] private static partial Regex BoldTagRegex(); [GeneratedRegex(@"]*>", RegexOptions.Compiled)] private static partial Regex AnchorTagRegex(); [GeneratedRegex(@"<[^>]*>", RegexOptions.Compiled)] private static partial Regex AllHtmlTagsRegex(); #endregion Genius Provider } } ================================================ FILE: Tubifarry/Metadata/Lyrics/TrackFileRepositoryHelper.cs ================================================ using Dapper; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.Extras.Lyrics; using NzbDrone.Core.Extras.Others; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Lyrics { /// /// Repository helper for querying track files and managing lyric file database entries. /// Provides efficient batch querying to avoid loading millions of records into memory. /// public sealed class TrackFileRepositoryHelper : BasicRepository { private readonly ITrackRepository _trackRepository; private readonly ILyricFileService _lyricFileService; private readonly IExtraFileService _otherExtraFileService; private readonly Logger _logger; public TrackFileRepositoryHelper( IMainDatabase database, IEventAggregator eventAggregator, ITrackRepository trackRepository, ILyricFileService lyricFileService, IExtraFileService otherExtraFileService, Logger logger) : base(database, eventAggregator) { _trackRepository = trackRepository; _lyricFileService = lyricFileService; _otherExtraFileService = otherExtraFileService; _logger = logger; } /// /// Gets the total count of track files without LRC files. /// public int GetTracksWithoutLrcFilesCount() { try { SqlBuilder builder = Builder() .Join((tf, t) => tf.Id == t.TrackFileId) .Join((t, r) => t.AlbumReleaseId == r.Id) .Join((r, a) => r.AlbumId == a.Id) .Join((album, artist) => album.ArtistMetadataId == artist.ArtistMetadataId) .LeftJoin(@"""LyricFiles"" ON ""TrackFiles"".""Id"" = ""LyricFiles"".""TrackFileId""") .Where(a => a.Monitored == true) .Where(r => r.Monitored == true) .Where(@"""LyricFiles"".""Id"" IS NULL") .SelectCount(); SqlBuilder.Template template = builder.AddPageCountTemplate(typeof(TrackFile)); using System.Data.IDbConnection conn = _database.OpenConnection(); return conn.ExecuteScalar(template.RawSql, template.Parameters); } catch (Exception ex) { _logger.Error(ex, "Error counting tracks without LRC files"); return 0; } } /// /// Queries track files without LRC files in batches using SQL LIMIT/OFFSET. /// /// Starting position in the result set /// Maximum number of records to return /// List of track files for this batch public List GetTracksWithoutLrcFilesBatch(int offset, int limit) { try { // Build SQL query with LIMIT/OFFSET for efficient pagination SqlBuilder builder = Builder() .Join((tf, t) => tf.Id == t.TrackFileId) .Join((t, r) => t.AlbumReleaseId == r.Id) .Join((r, a) => r.AlbumId == a.Id) .Join((album, artist) => album.ArtistMetadataId == artist.ArtistMetadataId) .LeftJoin(@"""LyricFiles"" ON ""TrackFiles"".""Id"" = ""LyricFiles"".""TrackFileId""") .Where(a => a.Monitored == true) .Where(r => r.Monitored == true) .Where(@"""LyricFiles"".""Id"" IS NULL") .GroupBy(tf => tf.Id) .OrderBy($@"""TrackFiles"".""Id"" ASC LIMIT {limit} OFFSET {offset}"); List trackFiles = Query(builder); foreach (TrackFile trackFile in trackFiles) { List tracks = _trackRepository.GetTracksByFileId(trackFile.Id); trackFile.Tracks = new LazyLoaded>(tracks); if (tracks.Count > 0 && tracks[0].Artist?.Value != null) { trackFile.Artist = new LazyLoaded(tracks[0].Artist.Value); } } return trackFiles; } catch (Exception ex) { _logger.Error(ex, $"Error querying tracks without LRC files (offset: {offset}, limit: {limit})"); return []; } } public LyricFile? CreateAndUpsertLyricFile(Artist artist, TrackFile trackFile, string relativePath) { try { OtherExtraFile? conflictingEntry = _otherExtraFileService.FindByPath(artist.Id, relativePath); if (conflictingEntry != null) { _logger.Warn($"Found .lrc file registered as OtherExtraFile (ID: {conflictingEntry.Id}), deleting conflicting entry to prevent deletion during re-imports: {relativePath}"); _otherExtraFileService.Delete(conflictingEntry.Id); } LyricFile lyricFile = new() { ArtistId = artist.Id, TrackFileId = trackFile.Id, AlbumId = trackFile.AlbumId, RelativePath = relativePath, Added = DateTime.UtcNow, LastUpdated = DateTime.UtcNow, Extension = ".lrc" }; _lyricFileService.Upsert(lyricFile); _logger.Debug($"Created and upserted lyric file to database: {relativePath}"); return lyricFile; } catch (Exception ex) { _logger.Error(ex, $"Failed to create and upsert lyric file: {relativePath}"); return null; } } /// /// Exposes the base Builder method for advanced queries. /// public new SqlBuilder Builder() => base.Builder(); /// /// Exposes the base Query method for advanced queries. /// public new List Query(SqlBuilder builder) => base.Query(builder); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/AlbumMapper.cs ================================================ using DryIoc.ImTools; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using NzbDrone.Core.Profiles.Metadata; using System.Globalization; namespace Tubifarry.Metadata.Proxy.MetadataProvider { public static class AlbumMapper { /// /// Primary type mapping: maps strings to a standard primary album type. /// public static readonly Dictionary PrimaryTypeMap = new(StringComparer.OrdinalIgnoreCase) { { "album", "Album" }, { "broadcast", "Broadcast" }, { "ep", "EP" }, { "single", "Single" }, // Discogs-specific format variations { "maxi-single", "Single" }, { "mini-album", "EP" }, { "maxisingle", "Single" } }; /// /// Secondary type keywords used for determining types based on title. /// public static readonly Dictionary> SecondaryTypeKeywords = new() { { SecondaryAlbumType.Live, new List { "live at", "live in", "live from", "in concert", "unplugged", "live performance", "live session", "recorded live", "live", "concert", "tour", "festival", "performance" } }, { SecondaryAlbumType.Remix, new List { "remixed", "remixes", "remastered", "remix", "remaster", "rework", "reimagined", "revisited", "redux", "deluxe edition", "expanded edition" } }, { SecondaryAlbumType.Compilation, new List { "greatest hits", "best of", "the best of", "very best", "ultimate", "essential", "essentials", "definitive", "collection", "anthology", "retrospective", "complete", "selected works", "treasures", "favorites", "favourites", " hits", "singles collection" } }, { SecondaryAlbumType.Soundtrack, new List { "original soundtrack", "original motion picture soundtrack", "music from and inspired by", "music from the motion picture", "music from the film", "original score", "film score", "movie soundtrack", "soundtrack", "ost", "music from", "inspired by the" } }, { SecondaryAlbumType.Spokenword, new List { "spoken word", "poetry reading", "lecture", "speech", "reading", "poetry" } }, { SecondaryAlbumType.Interview, new List { "interview", "interviews", "in conversation", "q&a", "conversation with" } }, { SecondaryAlbumType.Audiobook, new List { "audiobook", "audio book", "unabridged", "abridged", "narrated by", "narration" } }, { SecondaryAlbumType.Demo, new List { "demo", "demos", "rough mix", "rough mixes", "early recordings", "sessions", "outtakes", "alternate takes", "rarities", "unreleased", "bootleg" } }, { SecondaryAlbumType.Mixtape, new List { "mixtape", "mix tape", "the mixtape" } }, { SecondaryAlbumType.DJMix, new List { "dj mix", "continuous mix", "mixed by", "mix session", "live mix", "club mix" } }, { SecondaryAlbumType.Audiodrama, new List { "audio drama", "audio play", "radio play", "radio drama", "theater production", "theatre production" } } }; /// /// Direct mapping of strings to SecondaryAlbumType (used by Discogs). /// public static readonly Dictionary SecondaryTypeMap = new(StringComparer.OrdinalIgnoreCase) { { "compilation", SecondaryAlbumType.Compilation }, { "studio", SecondaryAlbumType.Studio }, { "soundtrack", SecondaryAlbumType.Soundtrack }, { "spokenword", SecondaryAlbumType.Spokenword }, { "interview", SecondaryAlbumType.Interview }, { "live", SecondaryAlbumType.Live }, { "remix", SecondaryAlbumType.Remix }, { "dj mix", SecondaryAlbumType.DJMix }, { "mixtape", SecondaryAlbumType.Mixtape }, { "demo", SecondaryAlbumType.Demo }, { "audio drama", SecondaryAlbumType.Audiodrama }, { "master", new() { Id = 36, Name = "Master" } }, { "release", new() { Id = 37, Name = "Release" } }, }; /// /// Extracts a link name from a URL. /// /// The URL to extract the link name from. /// A human-readable name for the link (e.g., "Bandcamp", "YouTube"). public static string GetLinkNameFromUrl(string url) { if (string.IsNullOrWhiteSpace(url)) return "Website"; try { Uri uri = new(url); string[] hostParts = uri.Host.Split('.', StringSplitOptions.RemoveEmptyEntries); if (hostParts.Contains("bandcamp")) return "Bandcamp"; if (hostParts.Contains("facebook")) return "Facebook"; if (hostParts.Contains("youtube")) return "YouTube"; if (hostParts.Contains("soundcloud")) return "SoundCloud"; if (hostParts.Contains("discogs")) return "Discogs"; string mainDomain = hostParts.Length > 1 ? hostParts[^2] : hostParts[0]; return mainDomain.ToUpper(CultureInfo.InvariantCulture); } catch { return "Website"; } } /// /// Determines secondary album types from a title using keyword matching. /// /// The title of the album to analyze. /// A list of detected secondary album types. Returns empty list if title is null or whitespace. public static List DetermineSecondaryTypesFromTitle(string title) { if (string.IsNullOrWhiteSpace(title)) return []; string? cleanTitle = Parser.NormalizeTitle(title)?.ToLowerInvariant(); if (cleanTitle == null) return []; title = title.ToLowerInvariant(); HashSet detectedTypes = []; HashSet keywordMatcher = []; foreach (KeyValuePair> kvp in SecondaryTypeKeywords) { keywordMatcher.Clear(); foreach (string keyword in kvp.Value) { keywordMatcher.Add(keyword.ToLowerInvariant()); } foreach (string keyword in keywordMatcher) { if (title.Contains(keyword) || cleanTitle.Contains(keyword)) { detectedTypes.Add(kvp.Key); break; } } } if (detectedTypes.Contains(SecondaryAlbumType.Live) && detectedTypes.Contains(SecondaryAlbumType.Remix)) detectedTypes.Remove(SecondaryAlbumType.Remix); return [.. detectedTypes]; } /// /// Filters albums based on a metadata profile. /// /// The collection of albums to filter. /// The ID of the metadata profile to use for filtering. /// The service to retrieve metadata profiles. /// A filtered collection of albums that match the metadata profile criteria. public static List FilterAlbums(IEnumerable albums, int metadataProfileId, IMetadataProfileService metadataProfileService) { MetadataProfile metadataProfile = metadataProfileService.Exists(metadataProfileId) ? metadataProfileService.Get(metadataProfileId) : metadataProfileService.All().First(); List primaryTypes = new(metadataProfile.PrimaryAlbumTypes.Where(s => s.Allowed).Select(s => s.PrimaryAlbumType.Name)); List secondaryTypes = new(metadataProfile.SecondaryAlbumTypes.Where(s => s.Allowed).Select(s => s.SecondaryAlbumType.Name)); List releaseStatuses = new(metadataProfile.ReleaseStatuses.Where(s => s.Allowed).Select(s => s.ReleaseStatus.Name)); NzbDroneLogger.GetLogger(typeof(AlbumMapper)).Trace($"Metadata Profile allows: Primary Types: {string.Join(", ", primaryTypes)} | Secondary Types: {string.Join(", ", secondaryTypes)} | Release Statuses: {string.Join(", ", releaseStatuses)}"); return albums.Where(album => primaryTypes.Contains(album.AlbumType) && ((album.SecondaryTypes.Count == 0 && secondaryTypes.Contains("Studio")) || album.SecondaryTypes.Any(x => secondaryTypes.Contains(x.Name))) && album.AlbumReleases.Value.Any(x => releaseStatuses.Contains(x.Status))).ToList(); } /// /// Maps album types based on a collection of format description strings. /// Sets both the primary album type and the secondary types. /// /// The collection of format descriptions to analyze. /// The album object to update with the mapped types. public static void MapAlbumTypes(IEnumerable? formatDescriptions, Album album) { album.AlbumType = "Album"; if (formatDescriptions != null) { foreach (string desc in formatDescriptions) { if (PrimaryTypeMap.TryGetValue(desc.ToLowerInvariant(), out string? primaryType)) { album.AlbumType = primaryType; break; } } HashSet secondaryTypes = []; foreach (string desc in formatDescriptions) { if (SecondaryTypeMap.TryGetValue(desc.ToLowerInvariant(), out SecondaryAlbumType? secondaryType)) secondaryTypes.Add(secondaryType); } album.SecondaryTypes = [.. secondaryTypes]; } } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/CustomLidarr/CustomLidarrMetadataProxy.cs ================================================ using NzbDrone.Common.Cloud; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using System.Text.RegularExpressions; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.MetadataProvider.CustomLidarr { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo))] [ProxyFor(typeof(IProvideAlbumInfo))] [ProxyFor(typeof(ISearchForNewArtist))] [ProxyFor(typeof(ISearchForNewAlbum))] [ProxyFor(typeof(ISearchForNewEntity))] [ProxyFor(typeof(IMetadataRequestBuilder))] public class CustomLidarrMetadataProxy : ProxyBase, IMetadata, ISupportMetadataMixing { private readonly ICustomLidarrProxy _customLidarrProxy; private readonly IConfigService _configService; private readonly ILidarrCloudRequestBuilder _defaultRequestFactory; public override string Name => "Lidarr Custom"; private static CustomLidarrMetadataProxySettings ActiveSettings => CustomLidarrMetadataProxySettings.Instance!; public CustomLidarrMetadataProxy(IConfigService configService, ILidarrCloudRequestBuilder defaultRequestBuilder, ICustomLidarrProxy customLidarrProxy) { _customLidarrProxy = customLidarrProxy; _configService = configService; _defaultRequestFactory = defaultRequestBuilder; } public List SearchForNewAlbum(string title, string artist) => _customLidarrProxy.SearchNewAlbum(ActiveSettings, title, artist); public List SearchForNewArtist(string title) => _customLidarrProxy.SearchNewArtist(ActiveSettings, title); public List SearchForNewEntity(string title) => _customLidarrProxy.SearchNewEntity(ActiveSettings, title); public Tuple> GetAlbumInfo(string foreignAlbumId) => _customLidarrProxy.GetAlbumInfo(ActiveSettings, foreignAlbumId); public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => _customLidarrProxy.GetArtistInfo(ActiveSettings, lidarrId, metadataProfileId); public HashSet GetChangedAlbums(DateTime startTime) => _customLidarrProxy.GetChangedAlbums(ActiveSettings, startTime); public HashSet GetChangedArtists(DateTime startTime) => _customLidarrProxy.GetChangedArtists(ActiveSettings, startTime); public List SearchForNewAlbumByRecordingIds(List recordingIds) => _customLidarrProxy.SearchNewAlbumByRecordingIds(ActiveSettings, recordingIds); public IHttpRequestBuilderFactory GetRequestBuilder() { if (Settings?.MetadataSource?.IsNotNullOrWhiteSpace() == true && ActiveSettings.CanProxySpotify) { return new HttpRequestBuilder(Settings?.MetadataSource.TrimEnd("/") + "/{route}").KeepAlive().CreateFactory(); } else if (_configService.MetadataSource.IsNotNullOrWhiteSpace()) { return new HttpRequestBuilder(_configService.MetadataSource.TrimEnd("/") + "/{route}").KeepAlive().CreateFactory(); } else { return _defaultRequestFactory.Search; } } public MetadataSupportLevel CanHandleSearch(string? albumTitle, string? artistName) { if (CustomLidarrProxy.IsMbidQuery(albumTitle) || CustomLidarrProxy.IsMbidQuery(artistName)) return MetadataSupportLevel.Supported; if (albumTitle != null && _formatRegex.IsMatch(albumTitle) || (artistName != null && _formatRegex.IsMatch(artistName))) return MetadataSupportLevel.Unsupported; return MetadataSupportLevel.Supported; } public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) { return MetadataSupportLevel.Supported; } public MetadataSupportLevel CanHandleChanged() { return MetadataSupportLevel.Supported; } /// /// Examines the provided list of links and returns the MusicBrainz GUID if one is found. /// Recognizes URLs such as: /// https://musicbrainz.org/artist/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// https://musicbrainz.org/release/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// https://musicbrainz.org/recording/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// public string? SupportsLink(List links) { if (links == null || links.Count == 0) return null; foreach (Links link in links) { if (string.IsNullOrWhiteSpace(link.Url)) continue; Match match = _musicBrainzRegex.Match(link.Url); if (match.Success && match.Groups.Count > 1) return match.Groups[1].Value; } return null; } /// /// Checks if the given id is in MusicBrainz GUID format (and does not contain an '@'). /// Returns Supported if valid; otherwise, Unsupported. /// public MetadataSupportLevel CanHandleId(string id) { if (string.IsNullOrWhiteSpace(id) || id.Contains('@')) return MetadataSupportLevel.Unsupported; if (_guidRegex.IsMatch(id)) return MetadataSupportLevel.Supported; return MetadataSupportLevel.Unsupported; } private static readonly Regex _formatRegex = new(@"^\s*\w+:\s*\w+", RegexOptions.Compiled); private static readonly Regex _musicBrainzRegex = new( @"musicbrainz\.org\/(?:artist|release|recording)\/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex _guidRegex = new("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", RegexOptions.IgnoreCase | RegexOptions.Compiled); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/CustomLidarr/CustomLidarrMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Metadata.Proxy.MetadataProvider.CustomLidarr { public class CustomLidarrMetadataProxySettingsValidator : AbstractValidator { private static readonly string[] _restrictedDomains = ["musicbrainz.org", "lidarr.audio"]; public CustomLidarrMetadataProxySettingsValidator() { // Validate that Warning is checked RuleFor(x => x.UseAtOwnRisk) .Equal(true) .WithMessage("You must acknowledge that this feature could void support by the servarr team by checking the 'Warning' box."); // Validate URL format and restrictions RuleFor(x => x.MetadataSource) .NotEmpty() .WithMessage("Metadata Source URL is required.") .IsValidUrl() .WithMessage("Metadata Source must be a valid HTTP or HTTPS URL.") .Must(NotBeRestrictedDomain) .WithMessage("Official MusicBrainz and Lidarr API endpoints are not allowed. Please use alternative metadata sources."); } private static bool NotBeRestrictedDomain(string url) => !_restrictedDomains.Any(domain => url.Contains(domain, StringComparison.InvariantCultureIgnoreCase)); } public class CustomLidarrMetadataProxySettings : IProviderConfig { private static readonly CustomLidarrMetadataProxySettingsValidator Validator = new(); [FieldDefinition(0, Label = "Metadata Source", Type = FieldType.Url, Placeholder = "https://api.musicinfo.pro", Section = MetadataSectionType.Metadata, HelpText = "URL to a compatible MusicBrainz API instance.")] public string MetadataSource { get; set; } = string.Empty; [FieldDefinition(1, Label = "Proxy Spotify", Type = FieldType.Checkbox, HelpText = "Enable if this metadata source can handle Spotify API requests.")] public bool CanProxySpotify { get; set; } [FieldDefinition(99, Label = "Warning", Type = FieldType.Checkbox, HelpText = "Use at your own risk. This feature could void your Servarr support.")] public bool UseAtOwnRisk { get; set; } public CustomLidarrMetadataProxySettings() => Instance = this; public static CustomLidarrMetadataProxySettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/CustomLidarr/CustomLidarrProxy.cs ================================================ using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Exceptions; using NzbDrone.Core.MediaCover; using NzbDrone.Core.MetadataSource.SkyHook; using NzbDrone.Core.MetadataSource.SkyHook.Resource; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using NzbDrone.Core.Profiles.Metadata; using System.Net; using System.Text.RegularExpressions; namespace Tubifarry.Metadata.Proxy.MetadataProvider.CustomLidarr { public partial class CustomLidarrProxy : ICustomLidarrProxy { private readonly IHttpClient _httpClient; private readonly Logger _logger; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly IMetadataProfileService _metadataProfileService; private readonly ICached> _cache; private static readonly List NonAudioMedia = new() { "DVD", "DVD-Video", "Blu-ray", "HD-DVD", "VCD", "SVCD", "UMD", "VHS" }; private static readonly List SkippedTracks = new() { "[data track]" }; public CustomLidarrProxy(IHttpClient httpClient, IArtistService artistService, IAlbumService albumService, Logger logger, IMetadataProfileService metadataProfileService, ICacheManager cacheManager) { _httpClient = httpClient; _metadataProfileService = metadataProfileService; _artistService = artistService; _albumService = albumService; _cache = cacheManager.GetCache>(GetType()); _logger = logger; } public HashSet GetChangedArtists(CustomLidarrMetadataProxySettings settings, DateTime startTime) { DateTimeOffset startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc); HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "recent/artist") .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds()) .Build(); httpRequest.SuppressHttpError = true; HttpResponse httpResponse = _httpClient.Get(httpRequest); if (httpResponse.Resource.Limited) { return null!; } return new HashSet(httpResponse.Resource.Items); } public Artist GetArtistInfo(CustomLidarrMetadataProxySettings settings, string foreignArtistId, int metadataProfileId) { _logger.Debug("Getting Artist with LidarrAPI.MetadataID of {0}", foreignArtistId); HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "artist/" + foreignArtistId) .Build(); httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; HttpResponse httpResponse = _httpClient.Get(httpRequest); if (httpResponse.HasHttpError) { if (httpResponse.StatusCode == HttpStatusCode.NotFound) { throw new ArtistNotFoundException(foreignArtistId); } else if (httpResponse.StatusCode == HttpStatusCode.BadRequest) { throw new BadRequestException(foreignArtistId); } else { throw new HttpException(httpRequest, httpResponse); } } Artist artist = new() { Metadata = MapArtistMetadata(httpResponse.Resource) }; artist.CleanName = artist.Metadata.Value.Name.CleanArtistName(); artist.SortName = Parser.NormalizeTitle(artist.Metadata.Value.Name); artist.Albums = FilterAlbums(httpResponse.Resource.Albums, metadataProfileId) .Select(x => MapAlbum(x, null!)).ToList(); return artist; } public HashSet GetChangedAlbums(CustomLidarrMetadataProxySettings settings, DateTime startTime) { return _cache.Get("ChangedAlbums", () => GetChangedAlbumsUncached(settings, startTime), TimeSpan.FromMinutes(30)); } private HashSet GetChangedAlbumsUncached(CustomLidarrMetadataProxySettings settings, DateTime startTime) { DateTimeOffset startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc); HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "recent/album") .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds()) .Build(); httpRequest.SuppressHttpError = true; HttpResponse httpResponse = _httpClient.Get(httpRequest); if (httpResponse.Resource.Limited) { return null!; } return new HashSet(httpResponse.Resource.Items); } public IEnumerable FilterAlbums(IEnumerable albums, int metadataProfileId) { MetadataProfile metadataProfile = _metadataProfileService.Exists(metadataProfileId) ? _metadataProfileService.Get(metadataProfileId) : _metadataProfileService.All().First(); HashSet primaryTypes = new(metadataProfile.PrimaryAlbumTypes.Where(s => s.Allowed).Select(s => s.PrimaryAlbumType.Name)); HashSet secondaryTypes = new(metadataProfile.SecondaryAlbumTypes.Where(s => s.Allowed).Select(s => s.SecondaryAlbumType.Name)); HashSet releaseStatuses = new(metadataProfile.ReleaseStatuses.Where(s => s.Allowed).Select(s => s.ReleaseStatus.Name)); return albums.Where(album => primaryTypes.Contains(album.Type) && (!album.SecondaryTypes.Any() && secondaryTypes.Contains("Studio") || album.SecondaryTypes.Any(x => secondaryTypes.Contains(x))) && album.ReleaseStatuses.Any(x => releaseStatuses.Contains(x))); } public Tuple> GetAlbumInfo(CustomLidarrMetadataProxySettings settings, string foreignAlbumId) { _logger.Debug("Getting Album with LidarrAPI.MetadataID of {0}", foreignAlbumId); HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "album/" + foreignAlbumId) .Build(); httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; HttpResponse httpResponse = _httpClient.Get(httpRequest); if (httpResponse.HasHttpError) { if (httpResponse.StatusCode == HttpStatusCode.NotFound) { throw new AlbumNotFoundException(foreignAlbumId); } else if (httpResponse.StatusCode == HttpStatusCode.BadRequest) { throw new BadRequestException(foreignAlbumId); } else { throw new HttpException(httpRequest, httpResponse); } } List artists = httpResponse.Resource.Artists.ConvertAll(MapArtistMetadata); Dictionary artistDict = artists.ToDictionary(x => x.ForeignArtistId, x => x); Album album = MapAlbum(httpResponse.Resource, artistDict); album.ArtistMetadata = artistDict[httpResponse.Resource.ArtistId]; return new Tuple>(httpResponse.Resource.ArtistId, album, artists); } public List SearchNewArtist(CustomLidarrMetadataProxySettings settings, string title) { try { string lowerTitle = title.ToLowerInvariant(); if (IsMbidQuery(lowerTitle)) { string slug = lowerTitle.Split(':')[1].Trim(); bool isValid = Guid.TryParse(slug, out Guid searchGuid); if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace) || !isValid) { return new List(); } try { Artist existingArtist = _artistService.FindById(searchGuid.ToString()); if (existingArtist != null) { return new List { existingArtist }; } int metadataProfile = _metadataProfileService.All().First().Id; return new List { GetArtistInfo(settings, searchGuid.ToString(), metadataProfile) }; } catch (ArtistNotFoundException) { return new List(); } } HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "search") .AddQueryParam("type", "artist") .AddQueryParam("query", title.ToLower().Trim()) .Build(); HttpResponse> httpResponse = _httpClient.Get>(httpRequest); return httpResponse.Resource.SelectList(MapSearchResult); } catch (HttpException ex) { _logger.Warn(ex); throw new SkyHookException("Search for '{0}' failed. Unable to communicate with LidarrAPI. {1}", ex, title, ex.Message); } catch (WebException ex) { _logger.Warn(ex); throw new SkyHookException("Search for '{0}' failed. Unable to communicate with LidarrAPI. {1}", ex, title, ex.Message); } catch (Exception ex) { _logger.Warn(ex); throw new SkyHookException("Search for '{0}' failed. Invalid response received from LidarrAPI. {1}", ex, title, ex.Message); } } public List SearchNewAlbum(CustomLidarrMetadataProxySettings settings, string title, string artist) { try { string lowerTitle = title.ToLowerInvariant(); if (IsMbidQuery(lowerTitle)) { string slug = lowerTitle.Split(':')[1].Trim(); bool isValid = Guid.TryParse(slug, out Guid searchGuid); if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace) || !isValid) { return new List(); } try { Album existingAlbum = _albumService.FindById(searchGuid.ToString()); if (existingAlbum == null) { Tuple> data = GetAlbumInfo(settings, searchGuid.ToString()); Album album = data.Item2; album.Artist = _artistService.FindById(data.Item1) ?? new Artist { Metadata = data.Item3.Single(x => x.ForeignArtistId == data.Item1) }; return new List { album }; } existingAlbum.Artist = _artistService.GetArtist(existingAlbum.ArtistId); return new List { existingAlbum }; } catch (AlbumNotFoundException) { return new List(); } } HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "search") .AddQueryParam("type", "album") .AddQueryParam("query", title.ToLower().Trim()) .AddQueryParam("artist", artist.IsNotNullOrWhiteSpace() ? artist.ToLower().Trim() : string.Empty) .AddQueryParam("includeTracks", "1") .Build(); HttpResponse> httpResponse = _httpClient.Get>(httpRequest); return httpResponse.Resource.Select(MapSearchResult) .Where(x => x != null) .ToList()!; } catch (HttpException) { throw new SkyHookException("Search for '{0}' failed. Unable to communicate with LidarrAPI.", title); } catch (Exception ex) { _logger.Warn(ex, ex.Message); throw new SkyHookException("Search for '{0}' failed. Invalid response received from LidarrAPI.", title); } } public List SearchNewAlbumByRecordingIds(CustomLidarrMetadataProxySettings settings, List recordingIds) { IEnumerable ids = recordingIds.Where(x => x.IsNotNullOrWhiteSpace()).Distinct(); HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "search/fingerprint") .Build(); httpRequest.SetContent(ids.ToJson()); httpRequest.Headers.ContentType = "application/json"; HttpResponse> httpResponse = _httpClient.Post>(httpRequest); return httpResponse.Resource.Select(MapSearchResult) .Where(x => x != null) .ToList()!; } public List SearchNewEntity(CustomLidarrMetadataProxySettings settings, string title) { string lowerTitle = title.ToLowerInvariant(); if (IsMbidQuery(lowerTitle)) { List artist = SearchNewArtist(settings, lowerTitle); if (artist.Any()) { return new List { artist[0] }; } List album = SearchNewAlbum(settings, lowerTitle, null!); if (album.Any()) { Album? result = album.FirstOrDefault(x => x.AlbumReleases.Value.Any()); if (result != null) { return new List { result }; } else { return new List(); } } } try { HttpRequest httpRequest = GetRequestBuilder(settings).Create() .SetSegment("route", "search") .AddQueryParam("type", "all") .AddQueryParam("query", lowerTitle.Trim()) .Build(); HttpResponse> httpResponse = _httpClient.Get>(httpRequest); return httpResponse.Resource.Select(MapSearchResult) .Where(x => x != null) .ToList()!; } catch (HttpException) { throw new SkyHookException("Search for '{0}' failed. Unable to communicate with LidarrAPI.", title); } catch (Exception ex) { _logger.Warn(ex, ex.Message); throw new SkyHookException("Search for '{0}' failed. Invalid response received from LidarrAPI.", title); } } public string? ExtractMbid(string? query) { if (query.IsNullOrWhiteSpace()) { return null; } Match match = MusicBrainzRegex().Match(query); return match.Success ? match.Groups[1].Value : null; } private Artist MapSearchResult(ArtistResource resource) { Artist? artist = _artistService.FindById(resource.Id); artist ??= new Artist { Metadata = MapArtistMetadata(resource) }; return artist; } private Album? MapSearchResult(AlbumResource resource) { Dictionary artists = resource.Artists.Select(MapArtistMetadata).ToDictionary(x => x.ForeignArtistId, x => x); Artist? artist = _artistService.FindById(resource.ArtistId); artist ??= new Artist { Metadata = artists[resource.ArtistId] }; Album album = _albumService.FindById(resource.Id) ?? MapAlbum(resource, artists); album.Artist = artist; album.ArtistMetadata = artist.Metadata.Value; if (!album.AlbumReleases.Value.Any()) { return null; } return album; } private object? MapSearchResult(EntityResource resource) { if (resource.Artist != null) { return MapSearchResult(resource.Artist); } else if (resource.Album != null) { return MapSearchResult(resource.Album); } else { return null; } } private static Album MapAlbum(AlbumResource resource, Dictionary artistDict) { Album album = new() { ForeignAlbumId = resource.Id, OldForeignAlbumIds = resource.OldIds, Title = resource.Title, Overview = resource.Overview, Disambiguation = resource.Disambiguation, ReleaseDate = resource.ReleaseDate }; if (resource.Images != null) { album.Images = resource.Images.ConvertAll(MapImage); } album.AlbumType = resource.Type; album.SecondaryTypes = resource.SecondaryTypes.ConvertAll(MapSecondaryTypes); album.Ratings = MapRatings(resource.Rating); album.Links = resource.Links?.Select(MapLink).ToList(); album.Genres = resource.Genres; album.CleanTitle = album.Title.CleanArtistName(); if (resource.Releases != null) { album.AlbumReleases = resource.Releases.Select(x => MapRelease(x, artistDict)).Where(x => x.TrackCount > 0).ToList(); // Monitor the release with most tracks AlbumRelease? mostTracks = album.AlbumReleases.Value.MaxBy(x => x.TrackCount); if (mostTracks != null) { mostTracks.Monitored = true; } } else { album.AlbumReleases = new List(); } album.AnyReleaseOk = true; return album; } private static AlbumRelease MapRelease(ReleaseResource resource, Dictionary artistDict) { AlbumRelease release = new() { ForeignReleaseId = resource.Id, OldForeignReleaseIds = resource.OldIds, Title = resource.Title, Status = resource.Status, Label = resource.Label, Disambiguation = resource.Disambiguation, Country = resource.Country, ReleaseDate = resource.ReleaseDate }; // Get the complete set of media/tracks returned by the API, adding missing media if necessary List allMedia = resource.Media.ConvertAll(MapMedium); IEnumerable allTracks = resource.Tracks.Select(x => MapTrack(x, artistDict)); if (!allMedia.Any()) { foreach (int n in allTracks.Select(x => x.MediumNumber).Distinct()) { allMedia.Add(new Medium { Name = "Unknown", Number = n, Format = "Unknown" }); } } // Skip non-audio media IEnumerable audioMediaNumbers = allMedia.Where(x => !NonAudioMedia.Contains(x.Format)).Select(x => x.Number); // Get tracks on the audio media and omit any that are skipped release.Tracks = allTracks.Where(x => audioMediaNumbers.Contains(x.MediumNumber) && !SkippedTracks.Contains(x.Title)).ToList(); release.TrackCount = release.Tracks.Value.Count; // Only include the media that contain the tracks we have selected IEnumerable usedMediaNumbers = release.Tracks.Value.Select(track => track.MediumNumber); release.Media = allMedia.Where(medium => usedMediaNumbers.Contains(medium.Number)).ToList(); release.Duration = release.Tracks.Value.Sum(x => x.Duration); return release; } private static Medium MapMedium(MediumResource resource) => new() { Name = resource.Name, Number = resource.Position, Format = resource.Format }; private static Track MapTrack(TrackResource resource, Dictionary artistDict) => new() { ArtistMetadata = artistDict[resource.ArtistId], Title = resource.TrackName, ForeignTrackId = resource.Id, OldForeignTrackIds = resource.OldIds, ForeignRecordingId = resource.RecordingId, OldForeignRecordingIds = resource.OldRecordingIds, TrackNumber = resource.TrackNumber, AbsoluteTrackNumber = resource.TrackPosition, Duration = resource.DurationMs, MediumNumber = resource.MediumNumber }; private static ArtistMetadata MapArtistMetadata(ArtistResource resource) => new() { Name = resource.ArtistName, Aliases = resource.ArtistAliases, ForeignArtistId = resource.Id, OldForeignArtistIds = resource.OldIds, Genres = resource.Genres, Overview = resource.Overview, Disambiguation = resource.Disambiguation, Type = resource.Type, Status = MapArtistStatus(resource.Status), Ratings = MapRatings(resource.Rating), Images = resource.Images?.Select(MapImage).ToList(), Links = resource.Links?.Select(MapLink).ToList() }; private static ArtistStatusType MapArtistStatus(string status) { if (status == null) { return ArtistStatusType.Continuing; } if (status.Equals("ended", StringComparison.InvariantCultureIgnoreCase)) { return ArtistStatusType.Ended; } return ArtistStatusType.Continuing; } private static Ratings MapRatings(RatingResource rating) { if (rating == null) { return new Ratings(); } return new Ratings { Votes = rating.Count, Value = rating.Value }; } private static MediaCover MapImage(ImageResource arg) { return new MediaCover { Url = arg.Url, CoverType = MapCoverType(arg.CoverType) }; } private static Links MapLink(LinkResource arg) => new() { Url = arg.Target, Name = arg.Type }; private static MediaCoverTypes MapCoverType(string coverType) => coverType.ToLower() switch { "poster" => MediaCoverTypes.Poster, "banner" => MediaCoverTypes.Banner, "fanart" => MediaCoverTypes.Fanart, "cover" => MediaCoverTypes.Cover, "disc" => MediaCoverTypes.Disc, "logo" or "clearlogo" => MediaCoverTypes.Clearlogo, _ => MediaCoverTypes.Unknown, }; public static SecondaryAlbumType MapSecondaryTypes(string albumType) => albumType.ToLowerInvariant() switch { "compilation" => SecondaryAlbumType.Compilation, "soundtrack" => SecondaryAlbumType.Soundtrack, "spokenword" => SecondaryAlbumType.Spokenword, "interview" => SecondaryAlbumType.Interview, "audiobook" => SecondaryAlbumType.Audiobook, "live" => SecondaryAlbumType.Live, "remix" => SecondaryAlbumType.Remix, "dj-mix" => SecondaryAlbumType.DJMix, "mixtape/street" => SecondaryAlbumType.Mixtape, "demo" => SecondaryAlbumType.Demo, "audio drama" => SecondaryAlbumType.Audiodrama, _ => SecondaryAlbumType.Studio, }; public static bool IsMbidQuery(string? query) => MusicBrainzRegex().IsMatch(query ?? string.Empty); private static IHttpRequestBuilderFactory GetRequestBuilder(CustomLidarrMetadataProxySettings settings) => new HttpRequestBuilder(settings.MetadataSource.TrimEnd("/") + "/{route}").KeepAlive().CreateFactory(); [GeneratedRegex(@"\b(?:lidarr:|lidarrid:|mbid:|cl:|clid:|customlidarrid:|musicbrainz\.org/(?:artist|release|recording|release-group)/)([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex MusicBrainzRegex(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/CustomLidarr/ICustomLidarrProxy.cs ================================================ using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider.CustomLidarr { public interface ICustomLidarrProxy { List SearchNewAlbum(CustomLidarrMetadataProxySettings settings, string title, string artist); List SearchNewArtist(CustomLidarrMetadataProxySettings settings, string title); List SearchNewEntity(CustomLidarrMetadataProxySettings settings, string query); List SearchNewAlbumByRecordingIds(CustomLidarrMetadataProxySettings settings, List recordingIds); Tuple> GetAlbumInfo(CustomLidarrMetadataProxySettings settings, string foreignAlbumId); Artist GetArtistInfo(CustomLidarrMetadataProxySettings settings, string foreignArtistId, int metadataProfileId); HashSet GetChangedAlbums(CustomLidarrMetadataProxySettings settings, DateTime startTime); HashSet GetChangedArtists(CustomLidarrMetadataProxySettings settings, DateTime startTime); string? ExtractMbid(string? query); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerAPIService.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public class DeezerApiService { private readonly IHttpClient _httpClient; private readonly Logger _logger; private readonly ICircuitBreaker _circuitBreaker; private readonly string _userAgent; /// /// OAuth access token. When set, requests include access_token. /// public string? AuthToken { get; set; } public string BaseUrl { get; set; } = "https://api.deezer.com"; public int MaxRetries { get; set; } = 5; public int InitialRetryDelayMs { get; set; } = 1000; public int MaxPageLimit { get; set; } = 5; public int PageSize { get; set; } = 25; private readonly TimeSpan _rateLimit = TimeSpan.FromSeconds(0.5); public DeezerApiService(IHttpClient httpClient, string userAgent) { _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); _circuitBreaker = CircuitBreakerFactory.GetBreaker(this); _userAgent = userAgent; } /// /// Constructs an HTTP request builder for a given endpoint. /// Includes authentication parameters if set. /// private HttpRequestBuilder BuildRequest(string endpoint) { HttpRequestBuilder builder = new HttpRequestBuilder(BaseUrl).Resource(endpoint); if (!string.IsNullOrWhiteSpace(AuthToken)) builder.AddQueryParam("access_token", AuthToken); builder.AllowAutoRedirect = true; builder.SuppressHttpError = true; builder.Headers.Add("User-Agent", _userAgent); builder.WithRateLimit(_rateLimit.TotalSeconds); _logger.Trace($"Building request for endpoint: {endpoint}"); return builder; } /// /// Executes an HTTP request with retry logic for HTTP 429. /// private async Task ExecuteRequestWithRetryAsync(HttpRequestBuilder requestBuilder, int retryCount = 0) { try { if (_circuitBreaker.IsOpen) { _logger.Warn("Circuit breaker is open, skipping request to Deezer API"); return default; } HttpRequest request = requestBuilder.Build(); HttpResponse response = await _httpClient.GetAsync(request); if (response.StatusCode == HttpStatusCode.TooManyRequests) { if (retryCount >= MaxRetries) { _logger.Warn("Max retries reached due to rate limiting."); _circuitBreaker.RecordFailure(); return default; } int delayMs = InitialRetryDelayMs * (int)Math.Pow(2, retryCount); _logger.Warn($"Rate limit exceeded. Retrying in {delayMs}ms..."); await Task.Delay(delayMs); return await ExecuteRequestWithRetryAsync(requestBuilder, retryCount + 1); } if (response.StatusCode != HttpStatusCode.OK) { HandleErrorResponse(response); _circuitBreaker.RecordFailure(); return default; } using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); _circuitBreaker.RecordSuccess(); return jsonDoc.RootElement.Clone(); } catch (HttpException ex) { _logger.Warn($"API Error: {ex.Message}"); _circuitBreaker.RecordFailure(); return default; } } /// /// Logs error responses from the API. /// private void HandleErrorResponse(HttpResponse response) { try { using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); JsonElement root = jsonDoc.RootElement; string errorMessage = root.TryGetProperty("error", out JsonElement errorElem) && errorElem.TryGetProperty("message", out JsonElement msgElem) ? msgElem.GetString() ?? $"API Error: {response.StatusCode}" : $"API Error: {response.StatusCode}"; _logger.Warn(errorMessage); } catch (Exception ex) { _logger.Error(ex, $"Failed to parse API error response. Status Code: {response.StatusCode}"); } } /// /// Fetches paginated results from a Deezer endpoint. /// private async Task?> FetchPaginatedResultsAsync(HttpRequestBuilder requestBuilder, int maxPages, int itemsPerPage) { List results = []; int page = 0; bool hasNextPage = true; while (hasNextPage) { HttpRequestBuilder pagedRequest = requestBuilder .AddQueryParam("index", (page * itemsPerPage).ToString(), true) .AddQueryParam("limit", itemsPerPage.ToString(), true); JsonElement response = await ExecuteRequestWithRetryAsync(pagedRequest); if (response.TryGetProperty("data", out JsonElement dataElement)) { List? pageResults = JsonSerializer.Deserialize>(dataElement.GetRawText()); if (pageResults != null) results.AddRange(pageResults); } else { break; } hasNextPage = response.TryGetProperty("next", out JsonElement nextElement) && !string.IsNullOrWhiteSpace(nextElement.GetString()); if (page >= maxPages - 1) break; page++; } _logger.Trace($"Fetched {results.Count} results across {page + 1} pages."); return results; } // Generic helper for endpoints that return paginated lists. private async Task?> GetPaginatedDataAsync(string endpoint, int? maxPages = null) where T : MappingAgent { return MappingAgent.MapAgent(await FetchPaginatedResultsAsync(BuildRequest(endpoint), maxPages ?? MaxPageLimit, PageSize), _userAgent); } // Single-object fetch methods remain unchanged. public async Task GetAlbumAsync(long albumId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"album/{albumId}")); DeezerAlbum? album = response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()); return MappingAgent.MapAgent(album, _userAgent); } public async Task GetArtistAsync(long artistId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"artist/{artistId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetChartAsync(int chartId = 0) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"chart/{chartId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetEditorialAsync(long editorialId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"editorial/{editorialId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetGenreAsync(long genreId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"genre/{genreId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task?> ListGenresAsync() { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest("genre")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize>(response.GetRawText()), _userAgent); } public async Task GetPlaylistAsync(long playlistId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"playlist/{playlistId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetPodcastAsync(long podcastId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"podcast/{podcastId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetRadioAsync(long radioId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"radio/{radioId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetTrackAsync(long trackId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"track/{trackId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetUserAsync(long? userId = null) { string userSegment = userId.HasValue ? userId.Value.ToString() : "me"; JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"user/{userSegment}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task GetOptionsAsync() { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest("options")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } // --- Generic Search Methods --- private static readonly Dictionary SearchEndpointMapping = new() { { typeof(DeezerAlbum), "album" }, { typeof(DeezerArtist), "artist" }, { typeof(DeezerPlaylist), "playlist" }, { typeof(DeezerPodcast), "podcast" }, { typeof(DeezerRadio), "radio" }, { typeof(DeezerTrack), "track" }, { typeof(DeezerUser), "user" } }; public async Task?> SearchAsync(DeezerSearchParameter searchRequest, int? maxPages = null) { if (!SearchEndpointMapping.TryGetValue(typeof(T), out string? endpointSegment)) throw new InvalidOperationException($"No search endpoint mapping for type {typeof(T).Name}."); string endpoint = $"search/{endpointSegment}"; HttpRequestBuilder request = BuildRequest(endpoint); string q = BuildAdvancedSearchQuery(searchRequest); request.AddQueryParam("q", q); return await FetchPaginatedResultsAsync(request, maxPages ?? MaxPageLimit, PageSize); } // Alternatively, keep your advanced search method as is. private static string BuildAdvancedSearchQuery(DeezerSearchParameter search) { List parts = []; if (!string.IsNullOrWhiteSpace(search.Query)) parts.Add(search.Query.Trim()); if (!string.IsNullOrWhiteSpace(search.Artist)) parts.Add($"artist:\"{search.Artist.Trim()}\""); if (!string.IsNullOrWhiteSpace(search.Album)) parts.Add($"album:\"{search.Album.Trim()}\""); if (!string.IsNullOrWhiteSpace(search.Track)) parts.Add($"track:\"{search.Track.Trim()}\""); if (!string.IsNullOrWhiteSpace(search.Label)) parts.Add($"label:\"{search.Label.Trim()}\""); if (search.DurMin.HasValue) parts.Add($"dur_min:{search.DurMin.Value}"); if (search.DurMax.HasValue) parts.Add($"dur_max:{search.DurMax.Value}"); if (search.BpmMin.HasValue) parts.Add($"bpm_min:{search.BpmMin.Value}"); if (search.BpmMax.HasValue) parts.Add($"bpm_max:{search.BpmMax.Value}"); return string.Join(" ", parts); } public async Task?> SearchAsync(DeezerSearchParameter searchRequest, int? maxPages = null) { HttpRequestBuilder request = BuildRequest("search"); string q = BuildAdvancedSearchQuery(searchRequest); request.AddQueryParam("q", q); return await FetchPaginatedResultsAsync(request, maxPages ?? MaxPageLimit, PageSize); } // --- Generic Chart Method --- private static readonly Dictionary ChartEndpointMapping = new() { { typeof(DeezerAlbum), "chart/0/albums" }, { typeof(DeezerArtist), "chart/0/artists" }, { typeof(DeezerTrack), "chart/0/tracks" } }; public async Task?> GetChartDataAsync(int? maxPages = null) where T : MappingAgent { if (!ChartEndpointMapping.TryGetValue(typeof(T), out string? endpoint)) throw new InvalidOperationException($"No chart endpoint defined for type {typeof(T).Name}."); return await GetPaginatedDataAsync(endpoint, maxPages); } // --- Generic Artist Sub-Endpoints using static type mappings --- private static readonly Dictionary ArtistSubEndpointMapping = new() { { typeof(DeezerPlaylist), "playlists" }, { typeof(DeezerRadio), "radio" }, { typeof(DeezerArtist), "related" }, { typeof(DeezerAlbum), "albums" }, { typeof(DeezerTrack), "top" } }; public async Task?> GetArtistDataAsync(long artistId, int? maxPages = null) where T : MappingAgent { if (!ArtistSubEndpointMapping.TryGetValue(typeof(T), out string? subEndpoint)) throw new InvalidOperationException($"No artist sub endpoint defined for type {typeof(T).Name}."); string endpoint = $"artist/{artistId}/{subEndpoint}"; return await GetPaginatedDataAsync(endpoint, maxPages); } // --- Generic Album Sub-Endpoints using static type mappings --- private static readonly Dictionary AlbumSubEndpointMapping = new() { { typeof(DeezerTrack), "tracks" } }; public async Task?> GetAlbumDataAsync(long albumId, int? maxPages = null) where T : MappingAgent { if (!AlbumSubEndpointMapping.TryGetValue(typeof(T), out string? subEndpoint)) throw new InvalidOperationException($"No album sub endpoint defined for type {typeof(T).Name}."); string endpoint = $"album/{albumId}/{subEndpoint}"; return await GetPaginatedDataAsync(endpoint, maxPages); } // --- Generic Radio Endpoints --- public Task?> GetRadioListsAsync(int? maxPages = null) => GetPaginatedDataAsync("radio/lists", maxPages); public Task?> GetRadioTopAsync(int? maxPages = null) => GetPaginatedDataAsync("radio/top", maxPages); public Task?> GetRadioGenresAsync(int? maxPages = null) => GetPaginatedDataAsync("radio/genres", maxPages); public Task?> GetPlaylistRadioAsync(long playlistId, int? maxPages = null) => GetPaginatedDataAsync($"playlist/{playlistId}/radio", maxPages); public Task?> GetRadiosAsync(int? maxPages = null) => GetPaginatedDataAsync("radio", maxPages); public Task?> GetPlaylistTracksAsync(long playlistId, int? maxPages = null) => GetPaginatedDataAsync($"playlist/{playlistId}/tracks", maxPages); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerAuthService.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.Json; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { /// /// Handles authentication with Deezer via OAuth. /// public class DeezerAuthService { private readonly IHttpClient _httpClient; private readonly Logger _logger; public DeezerAuthService(IHttpClient httpClient) { _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); } /// /// Exchanges an OAuth code for an access token. /// public async Task AuthenticateAsync(string appId, string appSecret, string redirectUri, string code) { string requestUrl = $"https://connect.deezer.com/oauth/access_token.php?app_id={appId}&secret={appSecret}&code={code}&output=json"; HttpRequest request = new HttpRequestBuilder(requestUrl).Build(); HttpResponse response = await _httpClient.GetAsync(request); if (response.StatusCode == HttpStatusCode.OK) { using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); if (jsonDoc.RootElement.TryGetProperty("access_token", out JsonElement tokenElement)) { string? token = tokenElement.GetString(); _logger.Debug("Successfully authenticated with Deezer."); return token; } } _logger.Warn("Failed to authenticate with Deezer."); return null; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerMappingHelper.cs ================================================ using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaCover; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using Tubifarry.Core.Replacements; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public static class DeezerMappingHelper { private const string _identifier = "@deezer"; /// /// Enhanced mapping of Deezer album to internal Album model with comprehensive data handling. /// public static Album MapAlbumFromDeezerAlbum(DeezerAlbum dAlbum, Artist? artist = null) { Album album = new() { ForeignAlbumId = dAlbum.Id + _identifier, Title = dAlbum.Title ?? string.Empty, ReleaseDate = dAlbum.ReleaseDate, CleanTitle = dAlbum.Title.CleanArtistName(), Links = [], Genres = dAlbum.Genres?.Data?.Select(g => g.Name).ToList() ?? [], AlbumType = AlbumMapper.PrimaryTypeMap.TryGetValue(dAlbum.RecordType.ToLowerInvariant(), out string? mappedType) ? mappedType : "Album", SecondaryTypes = [], Ratings = new Ratings { Votes = dAlbum.Fans, Value = Math.Min(dAlbum.Fans > 0 ? (decimal)(dAlbum.Fans / 1000.0) : 0, 0) }, AnyReleaseOk = true, }; List overviewParts = []; if (!string.IsNullOrWhiteSpace(dAlbum.Label)) overviewParts.Add($"Label: {dAlbum.Label}"); if (dAlbum.ReleaseDate != DateTime.MinValue) overviewParts.Add($"Released: {dAlbum.ReleaseDate:yyyy-MM-dd}"); if (dAlbum.NumberOfTracks > 0) overviewParts.Add($"{dAlbum.NumberOfTracks} tracks"); if (!string.IsNullOrWhiteSpace(dAlbum.UPC)) overviewParts.Add($"UPC: {dAlbum.UPC}"); album.Overview = overviewParts.Count != 0 ? string.Join(" • ", overviewParts) : "Found on Deezer"; album.Images = []; foreach (string? url in new[] { dAlbum.CoverMedium, dAlbum.CoverBig }) if (!string.IsNullOrEmpty(url)) album.Images.Add(new MediaCover(MediaCoverTypes.Cover, url + $"?{FlexibleHttpDispatcher.UA_PARAM}={dAlbum.UserAgent}")); album.Links.Add(new Links { Url = dAlbum.Link, Name = "Deezer" }); album.Links.Add(new Links { Url = dAlbum.Share, Name = "Deezer Share" }); if (!string.IsNullOrEmpty(dAlbum.UPC)) album.Links.Add(new Links { Url = $"upc:{dAlbum.UPC}", Name = "UPC" }); List tracks = dAlbum.Tracks?.Data ?? []; List diskNumbers = [.. tracks.Select(t => t.DiskNumber).Distinct().Order()]; if (diskNumbers.Count == 0) diskNumbers.Add(1); AlbumRelease albumRelease = new() { ForeignReleaseId = dAlbum.Id + _identifier, Title = dAlbum.Title, ReleaseDate = dAlbum.ReleaseDate, Duration = dAlbum.Duration * 1000, Media = diskNumbers.ConvertAll(d => new Medium { Format = "Digital Media", Name = $"Disk {d}", Number = d }), Album = album, TrackCount = dAlbum.NumberOfTracks, Label = !string.IsNullOrWhiteSpace(dAlbum.Label) ? [dAlbum.Label] : [], Status = "Official" }; if (artist != null || dAlbum.Artist != null) { artist ??= MapArtistFromDeezerArtist(dAlbum.Artist); album.Artist = artist; album.ArtistMetadata = artist.Metadata; album.ArtistMetadataId = artist.ArtistMetadataId; } tracks = tracks.Select((x, index) => x with { TrackPosition = index + 1 }).ToList(); albumRelease.Tracks = tracks.ConvertAll(dTrack => MapTrack(dTrack, album, albumRelease, artist!)) ?? []; if (dAlbum.Contributors?.Count > 1) album.SecondaryTypes.Add(SecondaryAlbumType.Compilation); List titleTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(dAlbum.Title ?? string.Empty); album.SecondaryTypes.AddRange(titleTypes); album.AlbumReleases = new LazyLoaded>([albumRelease]); return album; } /// /// Enhanced artist mapping with multi-size image support. /// public static Artist MapArtistFromDeezerArtist(DeezerArtist dArtist) { Artist artist = new() { ForeignArtistId = dArtist.Id + _identifier, Name = dArtist.Name, SortName = dArtist.Name, CleanName = dArtist.Name.CleanArtistName() }; ArtistMetadata metadata = new() { ForeignArtistId = dArtist.Id + _identifier, Name = dArtist.Name, Overview = $"Artist \"{dArtist.Name}\" found on Deezer{(dArtist.NbAlbum > 0 ? $" with {dArtist.NbAlbum} albums" : "")}{(dArtist.NbAlbum > 0 && dArtist.NbFan > 0 ? " and" : "")}{(dArtist.NbFan > 0 ? $" {dArtist.NbFan} fans" : "")}.", Images = GetArtistImages(dArtist), Links = [ new() { Url = dArtist.Link, Name = "Deezer" }, new() { Url = dArtist.Share, Name = "Deezer Share" } ], Genres = [], Members = [], Aliases = [], Status = ArtistStatusType.Continuing, Type = string.Empty, Ratings = new Ratings() }; artist.Metadata = new LazyLoaded(metadata); return artist; } private static List GetArtistImages(DeezerArtist artist) { List images = []; foreach (string? url in new[] { artist.PictureMedium, artist.PictureBig }) if (!string.IsNullOrEmpty(url)) images.Add(new MediaCover(MediaCoverTypes.Poster, url + $"?{FlexibleHttpDispatcher.UA_PARAM}={artist.UserAgent}")); return images; } /// /// Enhanced track mapping with ISRC and explicit content handling. /// public static Track MapTrack(DeezerTrack dTrack, Album album, AlbumRelease albumRelease, Artist artist) => new() { ForeignTrackId = $"{dTrack.Id}{_identifier}", ForeignRecordingId = $"{dTrack.Id}{_identifier}", Title = dTrack.Title, Duration = dTrack.Duration * 1000, TrackNumber = dTrack.TrackPosition.ToString(), Explicit = dTrack.ExplicitContentLyrics is (int)ExplicitContent.Explicit or (int)ExplicitContent.PartiallyExplicit, Album = album, AbsoluteTrackNumber = dTrack.TrackPosition, ArtistMetadata = album.ArtistMetadata, AlbumRelease = albumRelease, Artist = artist, MediumNumber = dTrack.DiskNumber, Ratings = new Ratings() }; /// /// Merges album information if an existing album is found. /// public static Album MergeAlbums(Album existingAlbum, Album mappedAlbum) { if (existingAlbum == null) return mappedAlbum; existingAlbum.UseMetadataFrom(mappedAlbum); existingAlbum.Artist = mappedAlbum.Artist ?? existingAlbum.Artist; existingAlbum.ArtistMetadata = mappedAlbum.ArtistMetadata ?? existingAlbum.ArtistMetadata; existingAlbum.AlbumReleases = mappedAlbum.AlbumReleases ?? existingAlbum.AlbumReleases; return existingAlbum; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerMetadataProxy.cs ================================================ using NLog; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using System.Text.RegularExpressions; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo))] [ProxyFor(typeof(IProvideAlbumInfo))] [ProxyFor(typeof(ISearchForNewArtist))] [ProxyFor(typeof(ISearchForNewAlbum))] [ProxyFor(typeof(ISearchForNewEntity))] public partial class DeezerMetadataProxy : ProxyBase, IMetadata, ISupportMetadataMixing { private readonly IDeezerProxy _deezerProxy; private readonly Logger _logger; public override string Name => "Deezer"; private DeezerMetadataProxySettings ActiveSettings => Settings ?? DeezerMetadataProxySettings.Instance!; public DeezerMetadataProxy(IDeezerProxy deezerProxy, Logger logger) { _deezerProxy = deezerProxy; _logger = logger; } public List SearchForNewAlbum(string title, string artist) => _deezerProxy.SearchNewAlbum(ActiveSettings, title, artist); public List SearchForNewArtist(string title) => _deezerProxy.SearchNewArtist(ActiveSettings, title); public List SearchForNewEntity(string title) => _deezerProxy.SearchNewEntity(ActiveSettings, title); public Tuple> GetAlbumInfo(string foreignAlbumId) => _deezerProxy.GetAlbumInfoAsync(ActiveSettings, foreignAlbumId).GetAwaiter().GetResult(); public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => _deezerProxy.GetArtistInfoAsync(ActiveSettings, lidarrId, metadataProfileId).GetAwaiter().GetResult(); public HashSet GetChangedAlbums(DateTime startTime) { _logger.Warn("GetChangedAlbums: Deezer API does not support change tracking; returning empty set."); return []; } public HashSet GetChangedArtists(DateTime startTime) { _logger.Warn("GetChangedArtists: Deezer API does not support change tracking; returning empty set."); return []; } public List SearchForNewAlbumByRecordingIds(List recordingIds) { _logger.Warn("SearchNewAlbumByRecordingIds: Deezer API does not support fingerprint search; returning empty list."); return []; } public MetadataSupportLevel CanHandleSearch(string? albumTitle, string? artistName) { if (DeezerProxy.IsDeezerIdQuery(albumTitle) || DeezerProxy.IsDeezerIdQuery(artistName)) return MetadataSupportLevel.Supported; if ((albumTitle != null && FormatRegex().IsMatch(albumTitle)) || (artistName != null && FormatRegex().IsMatch(artistName))) return MetadataSupportLevel.Unsupported; return MetadataSupportLevel.ImplicitSupported; } public MetadataSupportLevel CanHandleId(string id) { if (id.EndsWith("@deezer")) return MetadataSupportLevel.Supported; else return MetadataSupportLevel.Unsupported; } public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) { return MetadataSupportLevel.Unsupported; } public MetadataSupportLevel CanHandleChanged() { return MetadataSupportLevel.Unsupported; } public string? SupportsLink(List links) { if (links == null || links.Count == 0) return null; foreach (Links link in links) { if (string.IsNullOrWhiteSpace(link.Url)) continue; Match match = DeezerRegex().Match(link.Url); if (match.Success && match.Groups.Count > 1) return match.Groups[1].Value; } return null; } [GeneratedRegex(@"deezer\.com\/(?:album|artist|track)\/(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled, "de-DE")] private static partial Regex DeezerRegex(); [GeneratedRegex(@"^\s*\w+:\s*\w+", RegexOptions.Compiled)] private static partial Regex FormatRegex(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public class DeezerMetadataProxySettingsValidator : AbstractValidator { public DeezerMetadataProxySettingsValidator() { // Validate PageNumber must be greater than 0 RuleFor(x => x.PageNumber) .GreaterThan(0) .WithMessage("Page number must be greater than 0."); // Validate PageSize must be greater than 0 RuleFor(x => x.PageSize) .GreaterThan(0) .WithMessage("Page size must be greater than 0."); // When using Permanent cache, require a valid CacheDirectory RuleFor(x => x.CacheDirectory) .Must((settings, path) => settings.RequestCacheType != (int)CacheType.Permanent || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(4) || DateTime.UtcNow - Tubifarry.LastStarted > TimeSpan.FromDays(5)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // Validate the User-Agent RuleFor(x => x.UserAgent) .Must(x => UserAgentValidator.Instance.IsAllowed(x)) .WithMessage("The provided User-Agent is not allowed. " + "Ensure it follows the format 'Name/Version' and avoids terms like: lidarr, bot, crawler or proxy."); } } public class DeezerMetadataProxySettings : IProviderConfig { private static readonly DeezerMetadataProxySettingsValidator Validator = new(); [FieldDefinition(1, Label = "User Agent", Section = MetadataSectionType.Metadata, Type = FieldType.Textbox, HelpText = "Specify a custom User-Agent to identify yourself. A User-Agent helps servers understand the software making the request. Use a unique identifier that includes a name and version. Avoid generic or suspicious-looking User-Agents to prevent blocking.", Placeholder = "Lidarr/1.0.0")] public string UserAgent { get; set; } = string.Empty; [FieldDefinition(2, Label = "Page Number", Type = FieldType.Number, HelpText = "Page number for pagination", Placeholder = "1")] public int PageNumber { get; set; } = 1; [FieldDefinition(3, Label = "Page Size", Type = FieldType.Number, HelpText = "Page size for pagination", Placeholder = "10")] public int PageSize { get; set; } = 10; [FieldDefinition(4, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching")] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(5, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory to store cached data (only used for Permanent caching)")] public string CacheDirectory { get; set; } = string.Empty; public string BaseUrl => "https://api.deezer.com"; public DeezerMetadataProxySettings() => Instance = this; public static DeezerMetadataProxySettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerProxy.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Metadata; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public class DeezerProxy : IDeezerProxy { private const string _identifier = "@deezer"; private readonly Logger _logger; private readonly CacheService _cache; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly IHttpClient _httpClient; private readonly IMetadataProfileService _metadataProfileService; public DeezerProxy(Logger logger, IHttpClient httpClient, IArtistService artistService, IAlbumService albumService, IMetadataProfileService metadataProfileService) { _logger = logger; _httpClient = httpClient; _artistService = artistService; _albumService = albumService; _metadataProfileService = metadataProfileService; _cache = new CacheService(); } private void UpdateCache(DeezerMetadataProxySettings settings) { _cache.CacheDirectory = settings.CacheDirectory; _cache.CacheType = (CacheType)settings.RequestCacheType; } private async Task> CachedSearchAsync(DeezerMetadataProxySettings settings, string query, Func mapper, string? artist = null) { UpdateCache(settings); string key = $"{typeof(TSearch).Name}:{query}:{artist ?? ""}" + _identifier; List results = await _cache.FetchAndCacheAsync>(key, () => { DeezerApiService apiService = new(_httpClient, settings.UserAgent) { PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; return apiService.SearchAsync(new DeezerSearchParameter { Query = query, Artist = artist })!; }); return results.Select(r => mapper(r)).Where(x => x != null).ToList()!; } public async Task> GetArtistAlbumsAsync(DeezerMetadataProxySettings settings, Artist artist) { UpdateCache(settings); DeezerApiService apiService = new(_httpClient, settings.UserAgent); string artistAlbumsCacheKey = $"artist-albums:{artist.ForeignArtistId}" + _identifier; List albumResults = await _cache.FetchAndCacheAsync>( artistAlbumsCacheKey, () => apiService.GetArtistDataAsync(int.Parse(RemoveIdentifier(artist.ForeignArtistId)))!); List albums = []; foreach (DeezerAlbum albumD in albumResults) { if (albumD.Artist?.Id != null && RemoveIdentifier(artist.ForeignArtistId) != albumD.Artist.Id.ToString()) continue; Album album = DeezerMappingHelper.MapAlbumFromDeezerAlbum(albumD, artist); album = DeezerMappingHelper.MergeAlbums(_albumService.FindById(artist.ForeignArtistId), album); albums.Add(album); } return albums; } public List SearchNewAlbum(DeezerMetadataProxySettings settings, string title, string artist) { _logger.Debug($"SearchNewAlbum: title '{title}', artist '{artist}'"); UpdateCache(settings); try { return CachedSearchAsync(settings, title, item => { DeezerApiService apiService = new(_httpClient, settings.UserAgent) { PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; DeezerAlbum? albumDetails = apiService.GetAlbumAsync(item.Id).GetAwaiter().GetResult(); return albumDetails != null ? DeezerMappingHelper.MapAlbumFromDeezerAlbum(albumDetails) : null; }, artist).GetAwaiter().GetResult(); } catch (Exception ex) { _logger.Error($"SearchNewAlbum error: {ex}"); throw; } } public List SearchNewArtist(DeezerMetadataProxySettings settings, string title) { UpdateCache(settings); return CachedSearchAsync(settings, "", artistD => { Artist artist = DeezerMappingHelper.MapArtistFromDeezerArtist(artistD); artist.Albums = GetArtistAlbumsAsync(settings, artist).GetAwaiter().GetResult(); return artist; }, title).GetAwaiter().GetResult(); } public List SearchNewEntity(DeezerMetadataProxySettings settings, string query) { _logger.Debug($"SearchNewEntity invoked: query '{query}'"); UpdateCache(settings); query = SanitizeToUnicode(query); DeezerApiService apiService = new(_httpClient, settings.UserAgent) { PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; if (IsDeezerIdQuery(query)) { query = query.Replace("deezer:", "").Replace("deezerid:", ""); if (int.TryParse(query, out int deezerId)) { try { Artist artist = GetArtistInfoAsync(settings, deezerId.ToString(), 0).GetAwaiter().GetResult(); if (artist != null) return [artist]; } catch { } DeezerAlbum? albumResult = apiService.GetAlbumAsync(deezerId).GetAwaiter().GetResult(); if (albumResult != null) return [DeezerMappingHelper.MapAlbumFromDeezerAlbum(albumResult)]; } } List searchItems = CachedSearchAsync(settings, query, x => x).GetAwaiter().GetResult(); List artists = []; List results = []; foreach (DeezerAlbum? item in searchItems.DistinctBy(x => x.Id)) { Artist? mappedArtist = artists.Find(x => x.ForeignArtistId.StartsWith(item.Artist.Id.ToString())); if (mappedArtist == null) { mappedArtist = DeezerMappingHelper.MapArtistFromDeezerArtist(item.Artist); artists.Add(mappedArtist); results.Add(mappedArtist); } DeezerAlbum? albumDetails = item; Album mappedAlbum = DeezerMappingHelper.MapAlbumFromDeezerAlbum(albumDetails ?? item); mappedAlbum.Artist = mappedArtist; mappedAlbum.ArtistMetadata = mappedArtist.Metadata; results.Add(mappedAlbum); } return results; } public async Task>> GetAlbumInfoAsync(DeezerMetadataProxySettings settings, string foreignAlbumId) { _logger.Debug("Fetching album details for AlbumId: {0}", foreignAlbumId); UpdateCache(settings); DeezerApiService apiService = new(_httpClient, settings.UserAgent); string albumCacheKey = $"album:{foreignAlbumId}" + _identifier; Album? existingAlbum = _albumService.FindById(foreignAlbumId); DeezerAlbum albumDetails = await _cache.FetchAndCacheAsync(albumCacheKey, () => apiService.GetAlbumAsync(int.Parse(RemoveIdentifier(foreignAlbumId)))!) ?? throw new Exception("Album not found from Deezer API."); albumDetails = await EnsureAllTracksAsync(albumDetails, apiService); Artist? existingArtist = existingAlbum?.Artist?.Value ?? _artistService.FindById(albumDetails.Artist.Id + _identifier); if (existingArtist == null) { _logger.Debug("Artist {0} not found in database, mapping from Deezer data", albumDetails.Artist.Name); existingArtist = DeezerMappingHelper.MapArtistFromDeezerArtist(albumDetails.Artist); } Album mappedAlbum = DeezerMappingHelper.MapAlbumFromDeezerAlbum(albumDetails, existingArtist); Album finalAlbum = existingAlbum != null ? DeezerMappingHelper.MergeAlbums(existingAlbum, mappedAlbum) : mappedAlbum; _logger.Trace("Completed processing for AlbumId: {0}", foreignAlbumId); return new Tuple>(existingArtist.ForeignArtistId, finalAlbum, [existingArtist.Metadata.Value]); } private async Task EnsureAllTracksAsync(DeezerAlbum album, DeezerApiService apiService) { if (album.NumberOfTracks > 25 && (album.Tracks?.Data?.Count ?? 0) < album.NumberOfTracks) { string tracksCacheKey = $"album-tracks:{album.Id}" + _identifier; List? allTracks = await _cache.FetchAndCacheAsync>(tracksCacheKey, () => apiService.GetAlbumDataAsync(album.Id)!); if (allTracks != null && allTracks.Count > (album.Tracks?.Data?.Count ?? 0)) return album with { Tracks = new DeezerTrackWrapper(allTracks) }; } return album; } public async Task GetArtistInfoAsync(DeezerMetadataProxySettings settings, string foreignArtistId, int metadataProfileId) { _logger.Debug($"Fetching artist info for ID: {foreignArtistId}."); UpdateCache(settings); string artistCacheKey = $"artist:{foreignArtistId}" + _identifier; Artist? existingArtist = _artistService.FindById(foreignArtistId); if (existingArtist == null) { DeezerApiService apiService = new(_httpClient, settings.UserAgent); DeezerArtist? artistDetails = await _cache.FetchAndCacheAsync(artistCacheKey, () => apiService.GetArtistAsync(int.Parse(RemoveIdentifier(foreignArtistId)))!) ?? throw new KeyNotFoundException(); existingArtist = DeezerMappingHelper.MapArtistFromDeezerArtist(artistDetails); } existingArtist.Albums = AlbumMapper.FilterAlbums(await GetArtistAlbumsAsync(settings, existingArtist), metadataProfileId, _metadataProfileService); existingArtist.MetadataProfileId = metadataProfileId; _logger.Trace($"Processed artist: {existingArtist.Name} (ID: {existingArtist.ForeignArtistId})."); return existingArtist; } public static bool IsDeezerIdQuery(string? query) => query?.StartsWith("deezer:", StringComparison.OrdinalIgnoreCase) == true || query?.StartsWith("deezerid:", StringComparison.OrdinalIgnoreCase) == true; private static string SanitizeToUnicode(string input) => string.IsNullOrEmpty(input) ? input : new string(input.Where(c => c <= 0xFFFF).ToArray()); private static string RemoveIdentifier(string input) => input.EndsWith(_identifier, StringComparison.OrdinalIgnoreCase) ? input.Remove(input.Length - _identifier.Length) : input; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/DeezerRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public enum ExplicitContent { NotExplicit = 0, Explicit = 1, Unknown = 2, Edited = 3, PartiallyExplicit = 4, PartiallyUnknown = 5, NoAdviceAvailable = 6, PartiallyNoAdviceAvailable = 7 } public record DeezerAlbum( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("upc")] string? UPC, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("cover")] string Cover, [property: JsonPropertyName("cover_small")] string CoverSmall, [property: JsonPropertyName("cover_medium")] string CoverMedium, [property: JsonPropertyName("cover_big")] string CoverBig, [property: JsonPropertyName("cover_xl")] string CoverXL, [property: JsonPropertyName("md5_image")] string? Md5Image, [property: JsonPropertyName("genre_id")] int GenreId, [property: JsonPropertyName("genres")] DeezerGenresWrapper? Genres, [property: JsonPropertyName("label")] string? Label, [property: JsonPropertyName("nb_tracks")] int NumberOfTracks, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("fans")] int Fans, [property: JsonPropertyName("release_date")] DateTime ReleaseDate, [property: JsonPropertyName("record_type")] string RecordType, [property: JsonPropertyName("available")] bool Available, [property: JsonPropertyName("alternative")] DeezerAlbum? Alternative, [property: JsonPropertyName("tracklist")] string Tracklist, [property: JsonPropertyName("explicit_lyrics")] bool ExplicitLyrics, [property: JsonPropertyName("explicit_content_lyrics")] ExplicitContent ExplicitContentLyrics, [property: JsonPropertyName("explicit_content_cover")] ExplicitContent ExplicitContentCover, [property: JsonPropertyName("contributors")] List? Contributors, [property: JsonPropertyName("fallback")] DeezerFallbackAlbum? Fallback, [property: JsonPropertyName("artist")] DeezerArtist Artist, [property: JsonPropertyName("tracks")] DeezerTrackWrapper Tracks ) : MappingAgent; public record DeezerFallbackAlbum( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("status")] string Status ) : MappingAgent; public record DeezerArtist( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("picture")] string? Picture, [property: JsonPropertyName("picture_small")] string? PictureSmall, [property: JsonPropertyName("picture_medium")] string? PictureMedium, [property: JsonPropertyName("picture_big")] string? PictureBig, [property: JsonPropertyName("picture_xl")] string? PictureXL, [property: JsonPropertyName("nb_album")] int NbAlbum, [property: JsonPropertyName("nb_fan")] int NbFan, [property: JsonPropertyName("radio")] bool Radio, [property: JsonPropertyName("tracklist")] string Tracklist, [property: JsonPropertyName("position")] int? Position ) : MappingAgent; public record DeezerChart( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("tracks")] List? Tracks, [property: JsonPropertyName("albums")] List? Albums, [property: JsonPropertyName("artists")] List? Artists, [property: JsonPropertyName("playlists")] List? Playlists, [property: JsonPropertyName("podcasts")] List? Podcasts, [property: JsonPropertyName("position")] int? Position ) : MappingAgent; public record DeezerEditorial( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("picture")] string Picture, [property: JsonPropertyName("picture_small")] string PictureSmall, [property: JsonPropertyName("picture_medium")] string PictureMedium, [property: JsonPropertyName("picture_big")] string PictureBig, [property: JsonPropertyName("picture_xl")] string PictureXL ) : MappingAgent; public record DeezerEpisode( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("description")] string Description, [property: JsonPropertyName("available")] bool Available, [property: JsonPropertyName("release_date")] DateTime ReleaseDate, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("picture")] string Picture, [property: JsonPropertyName("picture_small")] string PictureSmall, [property: JsonPropertyName("picture_medium")] string PictureMedium, [property: JsonPropertyName("picture_big")] string PictureBig, [property: JsonPropertyName("picture_xl")] string PictureXL, [property: JsonPropertyName("podcast")] DeezerPodcast Podcast, [property: JsonPropertyName("track_token")] string TrackToken ) : MappingAgent; public record DeezerGenresWrapper( [property: JsonPropertyName("data")] List? Data ) : MappingAgent; public record DeezerGenre( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("picture")] string Picture, [property: JsonPropertyName("type")] string Type ) : MappingAgent; public record DeezerPlaylist( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("description")] string Description, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("public")] bool Public, [property: JsonPropertyName("is_loved_track")] bool IsLovedTrack, [property: JsonPropertyName("collaborative")] bool Collaborative, [property: JsonPropertyName("nb_tracks")] int NbTracks, [property: JsonPropertyName("unseen_track_count")] int UnseenTrackCount, [property: JsonPropertyName("fans")] int Fans, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("picture")] string? Picture, [property: JsonPropertyName("picture_small")] string? PictureSmall, [property: JsonPropertyName("picture_medium")] string? PictureMedium, [property: JsonPropertyName("picture_big")] string? PictureBig, [property: JsonPropertyName("picture_xl")] string? PictureXL, [property: JsonPropertyName("checksum")] string Checksum, [property: JsonPropertyName("creator")] DeezerUser Creator, [property: JsonPropertyName("tracks")] List? Tracks ) : MappingAgent; public record DeezerOptions( [property: JsonPropertyName("streaming")] bool Streaming, [property: JsonPropertyName("streaming_duration")] int StreamingDuration, [property: JsonPropertyName("offline")] bool Offline, [property: JsonPropertyName("hq")] bool HQ, [property: JsonPropertyName("ads_display")] bool AdsDisplay, [property: JsonPropertyName("ads_audio")] bool AdsAudio, [property: JsonPropertyName("too_many_devices")] bool TooManyDevices, [property: JsonPropertyName("can_subscribe")] bool CanSubscribe, [property: JsonPropertyName("radio_skips")] int RadioSkips, [property: JsonPropertyName("lossless")] bool Lossless, [property: JsonPropertyName("preview")] bool Preview, [property: JsonPropertyName("radio")] bool Radio ) : MappingAgent; public record DeezerPodcast( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("description")] string Description, [property: JsonPropertyName("available")] bool Available, [property: JsonPropertyName("fans")] int Fans, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("picture")] string Picture, [property: JsonPropertyName("picture_small")] string PictureSmall, [property: JsonPropertyName("picture_medium")] string PictureMedium, [property: JsonPropertyName("picture_big")] string PictureBig, [property: JsonPropertyName("picture_xl")] string PictureXL ) : MappingAgent; public record DeezerRadio( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("description")] string Description, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("picture")] string Picture, [property: JsonPropertyName("picture_small")] string PictureSmall, [property: JsonPropertyName("picture_medium")] string PictureMedium, [property: JsonPropertyName("picture_big")] string PictureBig, [property: JsonPropertyName("picture_xl")] string PictureXL, [property: JsonPropertyName("tracklist")] string Tracklist, [property: JsonPropertyName("md5_image")] string Md5Image ) : MappingAgent; public record DeezerTrackWrapper( [property: JsonPropertyName("data")] List? Data ) : MappingAgent; public record DeezerTrack( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("readable")] bool Readable, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("title_short")] string TitleShort, [property: JsonPropertyName("title_version")] string TitleVersion, [property: JsonPropertyName("unseen")] bool Unseen, [property: JsonPropertyName("isrc")] string ISRC, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("share")] string Share, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("track_position")] int TrackPosition, [property: JsonPropertyName("disk_number")] int DiskNumber, [property: JsonPropertyName("rank")] int Rank, [property: JsonPropertyName("release_date")] DateTime ReleaseDate, [property: JsonPropertyName("explicit_lyrics")] bool ExplicitLyrics, [property: JsonPropertyName("explicit_content_lyrics")] int ExplicitContentLyrics, [property: JsonPropertyName("explicit_content_cover")] int ExplicitContentCover, [property: JsonPropertyName("preview")] string? Preview, [property: JsonPropertyName("bpm")] double BPM, [property: JsonPropertyName("gain")] double Gain, [property: JsonPropertyName("available_countries")] List? AvailableCountries, [property: JsonPropertyName("alternative")] DeezerTrack? Alternative, [property: JsonPropertyName("contributors")] List? Contributors, [property: JsonPropertyName("md5_image")] string? Md5Image, [property: JsonPropertyName("track_token")] string TrackToken, [property: JsonPropertyName("artist")] DeezerArtist Artist, [property: JsonPropertyName("album")] DeezerAlbum Album, [property: JsonPropertyName("time_add")] long? TimeAddedInPlaylist ) : MappingAgent; public record DeezerUser( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("lastname")] string? LastName, [property: JsonPropertyName("firstname")] string? FirstName, [property: JsonPropertyName("email")] string? Email, [property: JsonPropertyName("status")] int? Status, [property: JsonPropertyName("birthday")] DateTime? Birthday, [property: JsonPropertyName("inscription_date")] DateTime InscriptionDate, [property: JsonPropertyName("gender")] string? Gender, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("picture")] string? Picture, [property: JsonPropertyName("picture_small")] string? PictureSmall, [property: JsonPropertyName("picture_medium")] string? PictureMedium, [property: JsonPropertyName("picture_big")] string? PictureBig, [property: JsonPropertyName("picture_xl")] string? PictureXL, [property: JsonPropertyName("country")] string Country, [property: JsonPropertyName("lang")] string? Lang, [property: JsonPropertyName("is_kid")] bool? IsKid, [property: JsonPropertyName("explicit_content_level")] string? ExplicitContentLevel, [property: JsonPropertyName("explicit_content_levels_available")] List? ExplicitContentLevelsAvailable, [property: JsonPropertyName("tracklist")] string Tracklist, [property: JsonPropertyName("role")] string? Role ) : MappingAgent; /// /// Represents the advanced search parameters for the Deezer API. /// Only non-null fields are included in the query. /// public record DeezerSearchParameter( string? Query = null, string? Artist = null, string? Album = null, string? Track = null, string? Label = null, int? DurMin = null, int? DurMax = null, int? BpmMin = null, int? BpmMax = null) { private static readonly Dictionary KeyMappings = new() { { nameof(Query), "q" }, { nameof(Artist), "artist" }, { nameof(Album), "album" }, { nameof(Track), "track" }, { nameof(Label), "label" }, { nameof(DurMin), "dur_min" }, { nameof(DurMax), "dur_max" }, { nameof(BpmMin), "bpm_min" }, { nameof(BpmMax), "bpm_max" } }; public Dictionary ToDictionary() => GetType().GetProperties() .Where(prop => prop.PropertyType == typeof(string) || prop.PropertyType == typeof(int?)) .Select(prop => (Key: KeyMappings[prop.Name], Value: prop.GetValue(this))) .Where(pair => pair.Value is not null) .ToDictionary(pair => pair.Key, pair => pair.Value!.ToString()!); } public record DeezerSearchItem( [property: JsonPropertyName("id")] long Id, [property: JsonPropertyName("readable")] bool Readable, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("title_short")] string TitleShort, [property: JsonPropertyName("title_version")] string TitleVersion, [property: JsonPropertyName("link")] string Link, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("rank")] int Rank, [property: JsonPropertyName("explicit_lyrics")] bool ExplicitLyrics, [property: JsonPropertyName("explicit_content_lyrics")] int ExplicitContentLyrics, [property: JsonPropertyName("explicit_content_cover")] int ExplicitContentCover, [property: JsonPropertyName("preview")] string Preview, [property: JsonPropertyName("md5_image")] string Md5Image, [property: JsonPropertyName("artist")] DeezerArtist Artist, [property: JsonPropertyName("album")] DeezerAlbum Album, [property: JsonPropertyName("type")] string Type ) : MappingAgent; } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Deezer/IDeezerProxy.cs ================================================ using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Deezer { public interface IDeezerProxy { List SearchNewAlbum(DeezerMetadataProxySettings settings, string title, string artist); List SearchNewArtist(DeezerMetadataProxySettings settings, string title); List SearchNewEntity(DeezerMetadataProxySettings settings, string query); Task>> GetAlbumInfoAsync(DeezerMetadataProxySettings settings, string foreignAlbumId); Task GetArtistInfoAsync(DeezerMetadataProxySettings settings, string foreignArtistId, int metadataProfileId); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsAPIService.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public class DiscogsApiService { private readonly IHttpClient _httpClient; private readonly Logger _logger; private readonly ICircuitBreaker _circuitBreaker; public string? AuthToken { get; set; } public string BaseUrl { get; set; } = "https://api.discogs.com"; public int MaxRetries { get; set; } = 5; public int InitialRetryDelayMs { get; set; } = 1000; public int MaxPageLimit { get; set; } = 5; public int PageSize { get; set; } = 30; private int _rateLimitTotal = 60; private int _rateLimitUsed; private int _rateLimitRemaining = 60; private DateTime _lastRequestTime = DateTime.MinValue; private readonly TimeSpan _rateLimit = TimeSpan.FromSeconds(0.75); private readonly string _userAgent; public DiscogsApiService(IHttpClient httpClient, string userAgent) { _httpClient = httpClient; _userAgent = userAgent; _logger = NzbDroneLogger.GetLogger(this); _circuitBreaker = CircuitBreakerFactory.GetBreaker(this); } public async Task GetReleaseAsync(int releaseId, string? currency = null) { HttpRequestBuilder request = BuildRequest($"releases/{releaseId}"); AddQueryParamIfNotNull(request, "curr_abbr", currency); JsonElement response = await ExecuteRequestWithRetryAsync(request); DiscogsRelease? release = response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()); return MappingAgent.MapAgent(release, _userAgent); } public async Task GetMasterReleaseAsync(int masterId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"masters/{masterId}")); DiscogsMasterRelease? master = response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()); return MappingAgent.MapAgent(master, _userAgent); } public async Task> GetMasterVersionsAsync(int masterId, int? maxPages = null, string? format = null, string? label = null, string? released = null, string? country = null, string? sort = null, string? sortOrder = null) { HttpRequestBuilder request = BuildRequest($"masters/{masterId}/versions"); AddQueryParamIfNotNull(request, "format", format); AddQueryParamIfNotNull(request, "label", label); AddQueryParamIfNotNull(request, "released", released); AddQueryParamIfNotNull(request, "country", country); AddQueryParamIfNotNull(request, "sort", sort); AddQueryParamIfNotNull(request, "sort_order", sortOrder); List masterReleaseVersions = await FetchPaginatedResultsAsync(request, maxPages ?? MaxPageLimit, PageSize) ?? []; return MappingAgent.MapAgent(masterReleaseVersions, _userAgent)!; } public async Task GetArtistAsync(int artistId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"artists/{artistId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task> GetArtistReleasesAsync(int artistId, int? maxPages = null, int? itemsPerPage = null, string? sort = null, string? sortOrder = null) { HttpRequestBuilder request = BuildRequest($"artists/{artistId}/releases"); AddQueryParamIfNotNull(request, "sort", sort); AddQueryParamIfNotNull(request, "sort_order", sortOrder); return MappingAgent.MapAgent(await FetchPaginatedResultsAsync(request, maxPages ?? MaxPageLimit, itemsPerPage ?? PageSize) ?? [], _userAgent)!; } public async Task GetLabelAsync(int labelId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"labels/{labelId}")); return MappingAgent.MapAgent(response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()), _userAgent); } public async Task> GetLabelReleasesAsync(int labelId, int? maxPages = null) { return MappingAgent.MapAgent(await FetchPaginatedResultsAsync(BuildRequest($"labels/{labelId}/releases"), maxPages ?? MaxPageLimit, PageSize) ?? [], _userAgent)!; } public async Task> SearchAsync(DiscogsSearchParameter searchRequest, int? maxPages = null) { HttpRequestBuilder request = BuildRequest("database/search"); AddSearchParams(request, searchRequest); return MappingAgent.MapAgent(await FetchPaginatedResultsAsync(request, maxPages ?? MaxPageLimit, PageSize) ?? [], _userAgent)!; } public async Task GetReleaseStatsAsync(int releaseId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"releases/{releaseId}/stats")); return response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()); } public async Task GetCommunityRatingAsync(int releaseId) { JsonElement response = await ExecuteRequestWithRetryAsync(BuildRequest($"releases/{releaseId}/rating")); return response.ValueKind == JsonValueKind.Undefined ? null : JsonSerializer.Deserialize(response.GetRawText()); } private HttpRequestBuilder BuildRequest(string? endpoint) { HttpRequestBuilder req = new HttpRequestBuilder(BaseUrl) .Resource(endpoint); if (!string.IsNullOrWhiteSpace(AuthToken)) req.SetHeader("Authorization", $"Discogs token={AuthToken}"); req.Headers.Add("User-Agent", _userAgent); req.AllowAutoRedirect = true; req.SuppressHttpError = true; _logger.Trace($"Building request for endpoint: {endpoint}"); return req; } private async Task ExecuteRequestWithRetryAsync(HttpRequestBuilder requestBuilder, int retryCount = 0) { try { if (_circuitBreaker.IsOpen) { _logger.Warn("Circuit breaker is open, skipping request to Deezer API"); return default; } await WaitForRateLimit(); requestBuilder.WithRateLimit(_rateLimit.TotalSeconds); HttpRequest request = requestBuilder.Build(); HttpResponse response = await _httpClient.GetAsync(request); UpdateRateLimitTracking(response); if (response.StatusCode == HttpStatusCode.TooManyRequests) { if (retryCount >= MaxRetries) { _logger.Warn("Max retries reached due to rate limiting."); _circuitBreaker.RecordFailure(); return default; } int delayMs = InitialRetryDelayMs * (int)Math.Pow(2, retryCount); _logger.Warn($"Rate limit exceeded. Retrying in {delayMs}ms..."); await Task.Delay(delayMs); return await ExecuteRequestWithRetryAsync(requestBuilder, retryCount + 1); } if (response.StatusCode != HttpStatusCode.OK) { HandleErrorResponse(response); _circuitBreaker.RecordFailure(); return default; } using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); _circuitBreaker.RecordSuccess(); return jsonDoc.RootElement.Clone(); } catch (HttpException ex) { _logger.Warn($"API Error: {ex.Message}"); return default; } } private async Task WaitForRateLimit() { if (_rateLimitRemaining <= 0) { TimeSpan timeSinceLastRequest = DateTime.UtcNow - _lastRequestTime; TimeSpan timeToWait = TimeSpan.FromSeconds(60) - timeSinceLastRequest; if (timeToWait > TimeSpan.Zero) { _logger.Debug($"Rate limit reached. Waiting for {timeToWait.TotalSeconds} seconds..."); await Task.Delay(timeToWait); } _rateLimitRemaining = _rateLimitTotal; _rateLimitUsed = 0; } } private void UpdateRateLimitTracking(HttpResponse response) { string? totalHeader = response.Headers.Get("X-Discogs-Ratelimit"); string? usedHeader = response.Headers.Get("X-Discogs-Ratelimit-Used"); string? remainingHeader = response.Headers.Get("X-Discogs-Ratelimit-Remaining"); if (!string.IsNullOrEmpty(totalHeader) && int.TryParse(totalHeader, out int total)) _rateLimitTotal = total; if (!string.IsNullOrEmpty(usedHeader) && int.TryParse(usedHeader, out int used)) _rateLimitUsed = used; if (!string.IsNullOrEmpty(remainingHeader) && int.TryParse(remainingHeader, out int remaining)) _rateLimitRemaining = remaining; _lastRequestTime = DateTime.UtcNow; _logger.Trace($"Rate limit updated - Total: {_rateLimitTotal}, Used: {_rateLimitUsed}, Remaining: {_rateLimitRemaining}"); } private async Task?> FetchPaginatedResultsAsync(HttpRequestBuilder requestBuilder, int? maxPages, int? itemsPerPage) { List results = []; int page = 1; bool hasNextPage = true; while (hasNextPage) { HttpRequestBuilder pagedRequest = requestBuilder .AddQueryParam("page", page.ToString(), true) .AddQueryParam("per_page", itemsPerPage.ToString(), true); JsonElement response = await ExecuteRequestWithRetryAsync(pagedRequest); if (response.TryGetProperty("results", out JsonElement resultsElement) || response.TryGetProperty("releases", out resultsElement)) { List? pageResults = JsonSerializer.Deserialize>(resultsElement.GetRawText()); if (pageResults != null) results.AddRange(pageResults); } else { break; } hasNextPage = response.TryGetProperty("pagination", out JsonElement pagination) && pagination.TryGetProperty("pages", out JsonElement pages) && pages.TryGetInt32(out int pagesInt) && pagesInt > page; if (maxPages.HasValue && page >= maxPages.Value) break; else if (page >= MaxPageLimit) break; page++; } _logger.Trace($"Fetched {results.Count} results across {page} pages."); return results; } private static void AddSearchParams(HttpRequestBuilder requestBuilder, DiscogsSearchParameter searchRequest) { foreach (KeyValuePair param in searchRequest.ToDictionary()) requestBuilder.AddQueryParam(param.Key, param.Value); } private void HandleErrorResponse(HttpResponse response) { try { using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); JsonElement root = jsonDoc.RootElement; string errorMessage = root.GetProperty("message").GetString() ?? $"API Error: {response.StatusCode}"; _logger.Warn(errorMessage); } catch (Exception ex) { _logger.Error(ex, $"Failed to parse API error response. Status Code: {response.StatusCode}"); } } private static HttpRequestBuilder AddQueryParamIfNotNull(HttpRequestBuilder request, string key, string? value) => value != null ? request.AddQueryParam(key, value) : request; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsMappingHelper.cs ================================================ using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaCover; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using System.Text; using System.Text.RegularExpressions; using Tubifarry.Core.Replacements; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public static partial class DiscogsMappingHelper { private const string _identifier = "@discogs"; private static readonly Dictionary FormatMap = new(StringComparer.OrdinalIgnoreCase) { ["Vinyl"] = "Vinyl", ["LP"] = "Vinyl", ["12\" Vinyl"] = "Vinyl", ["7\" Vinyl"] = "Vinyl", ["Cassette"] = "Cassette", ["Cass"] = "Cassette", ["CD"] = "CD", ["CDr"] = "CD-R", ["DVD"] = "DVD", ["Blu-ray"] = "Blu-ray", ["SACD"] = "SACD", ["Reel-To-Reel"] = "Reel to Reel", ["8-Track"] = "8-Track Cartridge", ["Flexi-disc"] = "Flexi Disc", ["Shellac"] = "Shellac", ["DAT"] = "DAT", ["MiniDisc"] = "MiniDisc", ["All Media"] = "Mixed Media", ["Box Set"] = "Box Set", ["Lathe Cut"] = "Lathe Cut", ["Acetate"] = "Acetate" }; private static string MapFormat(string discogsFormat) { if (string.IsNullOrWhiteSpace(discogsFormat)) return "Digital Media"; return FormatMap.TryGetValue(discogsFormat.Trim(), out string? mappedFormat) ? mappedFormat : "Digital Media"; } /// /// Parses a release date from a Discogs release. /// public static DateTime? ParseReleaseDate(DiscogsRelease release) { if (DateTime.TryParse(release.Released, out DateTime parsedDate)) return parsedDate; return ParseReleaseDate(release.Year); } /// /// Parses a release date from a Discogs artist release. /// public static DateTime? ParseReleaseDate(int? year) => year > 0 ? new DateTime(year ?? 0, 1, 1) : null; /// /// Analyzes a tracklist to determine the media/discs present. /// private static List DetermineMedia(List? tracklist, string defaultFormat = "Digital Media") { if (tracklist == null || tracklist.Count == 0) return [new() { Format = defaultFormat, Name = defaultFormat, Number = 1 }]; HashSet discNumbers = FilterTracklist(tracklist) .Select(t => ParseTrackPosition(t.Position, 1).DiscNumber) .ToHashSet(); if (discNumbers.Count == 0) return [new() { Format = defaultFormat, Name = defaultFormat, Number = 1 }]; return discNumbers.Order() .Select(n => new Medium { Format = defaultFormat, Name = $"{defaultFormat} {n}", Number = n }) .ToList(); } /// /// Parses a Discogs position string to extract disc and track numbers. /// Handles: 1-1, CD1-1, A1, AA1, a/aa, 1.1, 3.a, etc. /// private static DiscogsTrackPosition ParseTrackPosition(string? position, int sequentialNumber) { if (string.IsNullOrWhiteSpace(position)) return new DiscogsTrackPosition(1, sequentialNumber); Match match = PositionPattern().Match(position); if (!match.Success) return new DiscogsTrackPosition(1, sequentialNumber); string letters = match.Groups["letters"].Value; string first = match.Groups["first"].Value; string sep = match.Groups["sep"].Value; string second = match.Groups["second"].Value; // Multi-value format: "1-1", "CD1-1", "1.1" → (disc, track) if (!string.IsNullOrEmpty(sep) && int.TryParse(first, out int disc) && int.TryParse(second, out int track)) { // Sub-track exception: "3.a" keeps track 3 if (sep == "." && !char.IsDigit(second[0])) return new DiscogsTrackPosition(1, disc); return new DiscogsTrackPosition(disc, track); } // Letter-based formats: "A1", "AA1", "a", "aa" if (!string.IsNullOrEmpty(letters)) { bool hasTrack = int.TryParse(first, out int trackNum); // Lowercase repetition: "a"=1, "aa"=2, "aaa"=3 if (char.IsLower(letters[0]) && !hasTrack) return new DiscogsTrackPosition(1, letters.Length); // Vinyl with track: "A1", "AA1", "AB2" if (hasTrack) { // Box set continuation: "AA", "AB", "BA" = discs 14+ if (letters.Length == 2 && char.IsUpper(letters[0])) { int boxDisc = 13 + ((letters[0] - 'A') * 13) + (letters[1] - 'A') / 2 + 1; return new DiscogsTrackPosition(boxDisc, trackNum); } // Standard vinyl: "A1", "B2" return new DiscogsTrackPosition(1, trackNum); } // Letters only: "A", "AA" → use sequential return new DiscogsTrackPosition(1, sequentialNumber); } // Simple number: "1", "2", "3" return int.TryParse(first, out int simple) ? new DiscogsTrackPosition(1, simple) : new DiscogsTrackPosition(1, sequentialNumber); } /// /// Parses a duration string (e.g., "3:45") into seconds. /// public static int ParseDuration(string duration) { if (string.IsNullOrWhiteSpace(duration)) return 0; string[] parts = duration.Split(':'); if (parts.Length == 2 && int.TryParse(parts[0], out int m) && int.TryParse(parts[1], out int s)) return (m * 60) + s; return 0; } /// /// Maps a Discogs image to a MediaCover object. /// public static MediaCover? MapImage(DiscogsImage img, bool isArtist) => new() { Url = img.Uri, RemoteUrl = img.Uri, CoverType = MapCoverType(img.Type, isArtist) }; /// /// Maps a Discogs image type to a MediaCoverTypes enum. /// public static MediaCoverTypes MapCoverType(string? type, bool isArtist) { if (isArtist) { return type?.ToLowerInvariant() switch { "primary" or "avatar" => MediaCoverTypes.Poster, "banner" => MediaCoverTypes.Banner, "background" => MediaCoverTypes.Headshot, _ => MediaCoverTypes.Poster }; } return type?.ToLowerInvariant() switch { "primary" => MediaCoverTypes.Cover, "secondary" => MediaCoverTypes.Cover, _ => MediaCoverTypes.Unknown }; } /// /// Maps Discogs format descriptions to album types. /// public static void MapAlbumTypes(DiscogsRelease release, Album album) { List? formatDescriptions = release.Formats?.SelectMany(f => f.Descriptions ?? Enumerable.Empty()).ToList(); List? physicalFormats = release.Formats?.Select(f => f.Name).Where(n => !string.IsNullOrWhiteSpace(n)).ToList(); List filteredTracks = FilterTracklist(release.Tracklist); int trackCount = filteredTracks.Count; int totalDuration = filteredTracks.Sum(t => ParseDuration(t.Duration ?? "0")); // Tier 1: Standard format description matching AlbumMapper.MapAlbumTypes(formatDescriptions, album); if (album.AlbumType != "Album" || formatDescriptions?.Any(d => d.Equals("album", StringComparison.OrdinalIgnoreCase)) == true) return; // Tier 2: Physical format patterns string? inferredType = InferTypeFromPhysicalFormat(physicalFormats!, formatDescriptions); if (inferredType != null) { album.AlbumType = inferredType; return; } // Tier 3: Track count/duration heuristics album.AlbumType = InferTypeFromMetadata(trackCount, totalDuration, album.Title); } public static void MapAlbumTypes(DiscogsArtistRelease release, Album album) { List formatDescriptions = (release.Format ?? string.Empty).Split(',').Append(release.Type!).Select(f => f.Trim()).ToList(); // Tier 1: Standard format description matching AlbumMapper.MapAlbumTypes(formatDescriptions, album); if (album.AlbumType != "Album" || formatDescriptions.Any(d => d.Equals("album", StringComparison.OrdinalIgnoreCase))) return; // Tier 2: Physical format patterns List physicalFormats = (release.Format ?? string.Empty).Split(',').Select(f => f.Trim()).Where(f => !string.IsNullOrWhiteSpace(f)).ToList(); string? inferredType = InferTypeFromPhysicalFormat(physicalFormats!, formatDescriptions); if (inferredType != null) album.AlbumType = inferredType; } private static string? InferTypeFromPhysicalFormat(List? physicalFormats, List? formatDescriptions) { if (physicalFormats == null || !physicalFormats.Any()) return null; List allFormats = physicalFormats.Concat(formatDescriptions ?? Enumerable.Empty()).Select(f => f?.ToLowerInvariant() ?? string.Empty).ToList(); if (allFormats.Any(f => f.Contains("7\"") || f.Contains("7 inch"))) return "Single"; if (allFormats.Any(f => f.Contains("12\"") || f.Contains("12 inch"))) { if (allFormats.Any(f => f.Contains("ep") || f.Contains("mini"))) return "EP"; if (allFormats.Any(f => f.Contains("single sided") || f.Contains("promo"))) return "Single"; return null; } if (allFormats.Any(f => f == "lp" || f.Contains("long play"))) return "Album"; return null; } private static string InferTypeFromMetadata(int trackCount, int totalDurationSeconds, string? title) { if (!string.IsNullOrWhiteSpace(title)) { string lowerTitle = title.ToLowerInvariant(); if (lowerTitle.Contains(" ep") || lowerTitle.Contains("e.p.") || lowerTitle.EndsWith("ep")) return "EP"; if (lowerTitle.Contains("single") || (lowerTitle.Contains(" / ") && trackCount <= 3)) return "Single"; } int durationMinutes = totalDurationSeconds / 60; if (trackCount <= 3) return durationMinutes < 15 ? "Single" : "EP"; if (trackCount <= 7) return durationMinutes < 30 ? "EP" : "Album"; return "Album"; } /// /// Maps a DiscogsMasterRelease to an Album. Note that artist information is not set. /// public static Album MapAlbumFromMasterRelease(DiscogsMasterRelease masterRelease) { Album album = new() { ForeignAlbumId = "m" + masterRelease.Id + _identifier, Title = masterRelease.Title ?? string.Empty, ReleaseDate = ParseReleaseDate(masterRelease.Year), Genres = masterRelease.Genres != null || masterRelease.Styles != null ? new List(masterRelease.Genres ?? Enumerable.Empty()).Concat(masterRelease.Styles ?? Enumerable.Empty()).ToList() : [], CleanTitle = masterRelease.Title.CleanArtistName() ?? string.Empty, Overview = "Found on Discogs", Images = masterRelease.Images?.Take(2).Select(img => MapImage(img, false)).Where(x => x != null).ToList() ?? new List()!, Links = [new() { Url = masterRelease.ResourceUrl, Name = "Discogs" }], AlbumType = "Album", Ratings = new Ratings(), }; AlbumRelease albumRelease = new() { ForeignReleaseId = "m" + masterRelease.Id + _identifier, Title = masterRelease.Title, Status = "Official", Media = DetermineMedia(masterRelease.Tracklist), ReleaseDate = ParseReleaseDate(masterRelease.Year), }; album.AlbumReleases = new List { albumRelease }; album.AnyReleaseOk = true; album.SecondaryTypes = [AlbumMapper.SecondaryTypeMap["master"]]; List titleTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(masterRelease.Title ?? string.Empty); album.SecondaryTypes.AddRange(titleTypes); if (album.SecondaryTypes.Count == 1) album.SecondaryTypes.Add(SecondaryAlbumType.Studio); if (!string.IsNullOrEmpty(masterRelease.MainReleaseUrl)) album.Links.Add(new Links { Url = masterRelease.MainReleaseUrl, Name = "Main Release" }); if (!string.IsNullOrEmpty(masterRelease.VersionsUrl)) album.Links.Add(new Links { Url = masterRelease.VersionsUrl, Name = "Versions" }); return album; } /// /// Maps a detailed DiscogsRelease to an Album. Note that artist information is not set. /// public static Album MapAlbumFromRelease(DiscogsRelease release) { Album album = new() { ForeignAlbumId = "r" + release.Id + _identifier, Title = release.Title, ReleaseDate = ParseReleaseDate(release), Genres = release.Genres != null || release.Styles != null ? new List(release.Genres ?? Enumerable.Empty()).Concat(release.Styles ?? Enumerable.Empty()).ToList() : [], CleanTitle = release.Title.CleanArtistName() ?? string.Empty, Overview = release.Notes?.Trim(), Images = release.Images?.Take(2).Select(img => MapImage(img, false)).Where(x => x != null).ToList() ?? new List()!, Links = [new() { Url = release.ResourceUrl, Name = "Discogs" }], Ratings = ComputeCommunityRating(release.Community), SecondaryTypes = [], }; album.SecondaryTypes = [AlbumMapper.SecondaryTypeMap["release"]]; MapAlbumTypes(release, album); List titleTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(release.Title ?? string.Empty); album.SecondaryTypes.AddRange(titleTypes); album.SecondaryTypes = album.SecondaryTypes.DistinctBy(x => x.Id).ToList(); string formatName = release.Formats?.FirstOrDefault()?.Name ?? "Digital Media"; string mappedFormat = MapFormat(formatName); AlbumRelease albumRelease = new() { ForeignReleaseId = "r" + release.Id + _identifier, Title = release.Title ?? string.Empty, Status = release.Status ?? "Official", Media = DetermineMedia(release.Tracklist, mappedFormat), Label = release.Labels?.Select(l => l.Name).Where(l => !string.IsNullOrWhiteSpace(l)).ToList() ?? new List()!, ReleaseDate = ParseReleaseDate(release), Country = !string.IsNullOrWhiteSpace(release.Country) ? [release.Country] : [] }; album.AlbumReleases = new List { albumRelease }; album.AnyReleaseOk = true; return album; } /// /// Maps a DiscogsTrack to a Track object using data from a master release. /// public static Track MapTrack(DiscogsTrack t, DiscogsMasterRelease masterRelease, Album album, AlbumRelease albumRelease, int sequentialTrackNumber) { DiscogsTrackPosition position = ParseTrackPosition(t.Position, sequentialTrackNumber); return new Track { ForeignTrackId = $"m{masterRelease.Id + _identifier}_{t.Position}", Title = t.Title, Duration = ParseDuration(t.Duration ?? "0") * 1000, TrackNumber = position.TrackNumber.ToString(), Explicit = false, AlbumReleaseId = album.Id, ArtistMetadataId = album.ArtistMetadataId, Ratings = new Ratings(), ForeignRecordingId = $"m{masterRelease.Id + _identifier}_{t.Position}", Album = album, ArtistMetadata = album.ArtistMetadata, Artist = album.Artist, AlbumId = album.Id, AlbumRelease = albumRelease, MediumNumber = position.DiscNumber, AbsoluteTrackNumber = position.TrackNumber }; } /// /// Maps a Discogs track to a Track object. /// public static Track MapTrack(DiscogsTrack t, DiscogsRelease release, Album album, AlbumRelease albumRelease, int sequentialTrackNumber) { DiscogsTrackPosition position = ParseTrackPosition(t.Position, sequentialTrackNumber); return new Track { ForeignTrackId = $"r{release.Id + _identifier}_{t.Position}", Title = t.Title, Duration = ParseDuration(t.Duration ?? "0") * 1000, TrackNumber = position.TrackNumber.ToString(), Explicit = false, AlbumReleaseId = album.Id, ArtistMetadataId = album.ArtistMetadataId, Ratings = new Ratings(), ForeignRecordingId = $"r{release.Id + _identifier}_{t.Position}", Album = album, ArtistMetadata = album.ArtistMetadata, Artist = album.Artist, AlbumId = album.Id, AlbumRelease = albumRelease, MediumNumber = position.DiscNumber, AbsoluteTrackNumber = position.TrackNumber }; } /// /// Maps a DiscogsArtistRelease to an Album. This mapping does not include the artist. /// public static Album MapAlbumFromArtistRelease(DiscogsArtistRelease release) { Album album = new() { ForeignAlbumId = (release.Type == "master" ? "m" : "r") + release.Id + _identifier, Title = release.Title, Overview = release.Role ?? "Found on Discogs", ReleaseDate = ParseReleaseDate(release.Year), CleanTitle = release.Title.CleanArtistName() ?? string.Empty, Ratings = new Ratings(), Genres = [release.Label ?? string.Empty], Images = [new() { Url = release.Thumb + $"?{FlexibleHttpDispatcher.UA_PARAM}={release.UserAgent}" }], }; album.AlbumReleases = new List() { new () { Status = "Official", Album = album, Title = release.Title, Tracks = new List(), ForeignReleaseId = release.Id + _identifier } }; MapAlbumTypes(release, album); List titleTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(release.Title ?? string.Empty); album.SecondaryTypes.AddRange(titleTypes); if (album.SecondaryTypes.Count == 1) album.SecondaryTypes.Add(SecondaryAlbumType.Studio); return album; } /// /// Maps a DiscogsArtist to an Artist. /// public static Artist MapArtistFromDiscogsArtist(DiscogsArtist discogsArtist) => new() { Metadata = new ArtistMetadata() { Name = discogsArtist.Name ?? string.Empty, ForeignArtistId = "a" + discogsArtist.Id + _identifier, Aliases = discogsArtist.NameVariations ?? [], Images = discogsArtist.Images?.Select(img => MapImage(img, true)).ToList() ?? new List()!, Ratings = new Ratings(), Links = discogsArtist.Urls?.Select(url => new Links { Url = url, Name = AlbumMapper.GetLinkNameFromUrl(url) }).ToList() ?? [], Type = discogsArtist.Role ?? string.Empty, Genres = [], Overview = BuildArtistOverview(discogsArtist), Members = discogsArtist.Members?.Select(member => MapDiscogsMember(member)).ToList() ?? [], Status = discogsArtist.Members?.Any(x => x.Active) == false ? ArtistStatusType.Ended : ArtistStatusType.Continuing, }, Name = discogsArtist.Name, CleanName = discogsArtist.Name.CleanArtistName() }; public static Album MergeAlbums(Album existingAlbum, Album mappedAlbum) { if (existingAlbum == null) return mappedAlbum; existingAlbum.UseMetadataFrom(mappedAlbum); existingAlbum.Artist = mappedAlbum.Artist ?? existingAlbum.Artist; existingAlbum.ArtistMetadata = mappedAlbum.ArtistMetadata ?? existingAlbum.ArtistMetadata; existingAlbum.AlbumReleases = mappedAlbum.AlbumReleases ?? existingAlbum.AlbumReleases; return existingAlbum; } public static List MapTracks(object releaseForTracks, Album album, AlbumRelease albumRelease) { List? tracklist = releaseForTracks switch { DiscogsMasterRelease master => master.Tracklist, DiscogsRelease release => release.Tracklist, _ => null }; return FilterTracklist(tracklist).Select((track, index) => releaseForTracks switch { DiscogsMasterRelease master => MapTrack(track, master, album, albumRelease, index + 1), DiscogsRelease release => MapTrack(track, release, album, albumRelease, index + 1), _ => throw new InvalidOperationException("Invalid release type") }).ToList(); } private static List FilterTracklist(List? tracklist) => tracklist? .Where(t => !string.Equals(t.Type, "heading", StringComparison.OrdinalIgnoreCase) && !string.Equals(t.Type, "index", StringComparison.OrdinalIgnoreCase)) .ToList() ?? []; /// /// Creates a concise artist overview using DiscogsArtist data. /// private static string BuildArtistOverview(DiscogsArtist discogsArtist) { StringBuilder overview = new(); if (!string.IsNullOrEmpty(discogsArtist.Profile)) overview.AppendLine(discogsArtist.Profile); if (!string.IsNullOrEmpty(discogsArtist.Role) || !string.IsNullOrEmpty(discogsArtist.Join)) overview.AppendLine().AppendLine("Role and Involvement:").AppendLine($"- Role: {discogsArtist.Role ?? "Not specified"}").AppendLine($"- Joined: {discogsArtist.Join ?? "Not specified"}"); if (discogsArtist.NameVariations?.Any() == true) { overview.AppendLine().AppendLine("Name Variations:"); foreach (string variation in discogsArtist.NameVariations) overview.AppendLine($"- {variation}"); } if (!string.IsNullOrEmpty(discogsArtist.DataQuality)) overview.AppendLine().AppendLine($"Data Quality: {discogsArtist.DataQuality}"); return overview.ToString().Trim(); } /// /// Maps a DiscogsSearchItem to an Artist. /// public static Artist MapArtistFromSearchItem(DiscogsSearchItem searchItem) => new() { Metadata = new ArtistMetadata() { Name = searchItem.Title ?? string.Empty, ForeignArtistId = "a" + searchItem.Id + _identifier, Overview = "Found on Discogs", Images = [new() { Url = searchItem.Thumb + $"?{FlexibleHttpDispatcher.UA_PARAM}={searchItem.UserAgent}", CoverType = MapCoverType("primary", true) }], Links = [new() { Url = searchItem.ResourceUrl, Name = "Discogs" }], Ratings = ComputeCommunityRating(searchItem.Community), Genres = searchItem.Genre, } }; private static Member MapDiscogsMember(DiscogsMember discogsMember) => new() { Name = discogsMember.Name ?? string.Empty }; public static Ratings ComputeCommunityRating(DiscogsCommunityInfo? communityInfo) { if (communityInfo?.Rating != null) return new Ratings() { Value = Math.Truncate(communityInfo.Rating.Average), Votes = communityInfo.Rating.Count }; int want = communityInfo?.Want ?? 0; int have = communityInfo?.Have ?? 0; if (want == 0 && have == 0) return new Ratings { Value = 0m, Votes = 0 }; decimal smoothWant = want + 1; decimal smoothHave = have + 1; decimal ratio = smoothWant / smoothHave; decimal normalizedRatio = ratio / (ratio + 1); decimal proportion = smoothWant / (smoothWant + smoothHave); decimal computedValue = (0.7m * normalizedRatio) + (0.3m * proportion); decimal roundedValue = Math.Truncate(computedValue * 100m); return new Ratings { Value = roundedValue, Votes = want + have }; } // Regex: [format][letters][number][sep][number/letter] [GeneratedRegex(@"^(?:[A-Za-z]+(?=\d))?(?[A-Za-z]+)?(?\d+)?(?[-\.])?(?\d+|[a-z]+)?$", RegexOptions.Compiled)] private static partial Regex PositionPattern(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsMetadataProxy.cs ================================================ using NLog; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using System.Text.RegularExpressions; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo))] [ProxyFor(typeof(IProvideAlbumInfo))] [ProxyFor(typeof(ISearchForNewArtist))] [ProxyFor(typeof(ISearchForNewAlbum))] [ProxyFor(typeof(ISearchForNewEntity))] public partial class DiscogsMetadataProxy : ProxyBase, IMetadata, ISupportMetadataMixing { private readonly IDiscogsProxy _discogsProxy; private readonly Logger _logger; public override string Name => "Discogs"; private DiscogsMetadataProxySettings ActiveSettings => Settings ?? DiscogsMetadataProxySettings.Instance!; public DiscogsMetadataProxy(DiscogsProxy discogsProxy, Logger logger) { _discogsProxy = discogsProxy; _logger = logger; } public List SearchForNewAlbum(string title, string artist) => _discogsProxy.SearchNewAlbum(ActiveSettings, title, artist); public List SearchForNewArtist(string title) => _discogsProxy.SearchNewArtist(ActiveSettings, title); public List SearchForNewEntity(string title) => _discogsProxy.SearchNewEntity(ActiveSettings, title); public Tuple> GetAlbumInfo(string foreignAlbumId) => _discogsProxy.GetAlbumInfoAsync(ActiveSettings, foreignAlbumId).GetAwaiter().GetResult(); public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => _discogsProxy.GetArtistInfoAsync(ActiveSettings, lidarrId, metadataProfileId).GetAwaiter().GetResult(); public HashSet GetChangedAlbums(DateTime startTime) { _logger.Warn("GetChangedAlbums: Discogs API does not support change tracking; returning empty set."); return []; } public HashSet GetChangedArtists(DateTime startTime) { _logger.Warn("GetChangedArtists: Discogs API does not support change tracking; returning empty set."); return []; } public List SearchForNewAlbumByRecordingIds(List recordingIds) { _logger.Warn("SearchNewAlbumByRecordingIds: Discogs API does not support fingerprint search; returning empty list."); return []; } public MetadataSupportLevel CanHandleSearch(string? albumTitle, string? artistName) { if (_discogsProxy.IsDiscogsidQuery(albumTitle) || _discogsProxy.IsDiscogsidQuery(artistName)) return MetadataSupportLevel.Supported; if ((albumTitle != null && FormatRegex().IsMatch(albumTitle)) || (artistName != null && FormatRegex().IsMatch(artistName))) return MetadataSupportLevel.Unsupported; return MetadataSupportLevel.ImplicitSupported; } public MetadataSupportLevel CanHandleId(string id) { if (id.EndsWith("@discogs")) return MetadataSupportLevel.Supported; else return MetadataSupportLevel.Unsupported; } public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) => MetadataSupportLevel.Unsupported; public MetadataSupportLevel CanHandleChanged() => MetadataSupportLevel.Unsupported; public string? SupportsLink(List links) { if (links == null || links.Count == 0) return null; foreach (Links link in links) { if (string.IsNullOrWhiteSpace(link.Url)) continue; Match match = DiscogsRegex().Match(link.Url); if (match.Success && match.Groups.Count > 1) return match.Groups[1].Value; } return null; } [GeneratedRegex(@"^\s*\w+:\s*\w+", RegexOptions.Compiled)] private static partial Regex FormatRegex(); [GeneratedRegex(@"discogs\.com\/(?:artist|release|master)\/(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled, "de-DE")] private static partial Regex DiscogsRegex(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public class DiscogsMetadataProxySettingsValidator : AbstractValidator { public DiscogsMetadataProxySettingsValidator() { RuleFor(x => x.AuthToken).NotEmpty().WithMessage("A Discogs API key is required."); // Validate PageNumber must be greater than 0 RuleFor(x => x.PageNumber) .GreaterThan(0) .WithMessage("Page number must be greater than 0."); // Validate PageSize must be greater than 0 RuleFor(x => x.PageSize) .GreaterThan(0) .WithMessage("Page size must be greater than 0."); // When using Permanent cache, require a valid CacheDirectory. RuleFor(x => x.CacheDirectory) .Must((settings, path) => settings.RequestCacheType != (int)CacheType.Permanent || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(4) || DateTime.UtcNow - Tubifarry.LastStarted > TimeSpan.FromDays(5)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // Validate the User-Agent RuleFor(x => x.UserAgent) .Must(x => UserAgentValidator.Instance.IsAllowed(x)) .WithMessage("The provided User-Agent is not allowed." + "Ensure it follows the format 'Name/Version' and avoids terms like: lidarr, bot, crawler or proxy."); } } public class DiscogsMetadataProxySettings : IProviderConfig { private static readonly DiscogsMetadataProxySettingsValidator Validator = new(); [FieldDefinition(1, Label = "Token", Type = FieldType.Textbox, Privacy = PrivacyLevel.ApiKey, HelpText = "Your Discogs personal access token", Placeholder = "Enter your API key")] public string AuthToken { get; set; } = string.Empty; [FieldDefinition(2, Label = "User Agent", Section = MetadataSectionType.Metadata, Type = FieldType.Textbox, HelpText = "Specify a custom User-Agent to identify yourself. A User-Agent helps servers understand the software making the request. Use a unique identifier that includes a name and version. Avoid generic or suspicious-looking User-Agents to prevent blocking.", Placeholder = "Lidarr/1.0.0")] public string UserAgent { get; set; } = string.Empty; [FieldDefinition(3, Label = "Page Number", Type = FieldType.Number, HelpText = "Page number for pagination", Placeholder = "1")] public int PageNumber { get; set; } = 1; [FieldDefinition(4, Label = "Page Size", Type = FieldType.Number, HelpText = "Page size for pagination", Placeholder = "5")] public int PageSize { get; set; } = 5; [FieldDefinition(5, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching")] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(6, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory to store cached data (only used for Permanent caching)")] public string CacheDirectory { get; set; } = string.Empty; public string BaseUrl => "https://api.discogs.com"; public DiscogsMetadataProxySettings() => Instance = this; public static DiscogsMetadataProxySettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsProxy.cs ================================================ using FuzzySharp; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Datastore; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Metadata; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public class DiscogsProxy : IDiscogsProxy { private const string _identifier = "@discogs"; private readonly Logger _logger; private readonly CacheService _cache; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly IHttpClient _httpClient; private readonly IMetadataProfileService _metadataProfileService; private readonly IPluginSettings _pluginSettings; public DiscogsProxy(Logger logger, IPluginSettings pluginSettings, IHttpClient httpClient, IArtistService artistService, IAlbumService albumService, IMetadataProfileService metadataProfileService) { _logger = logger; _httpClient = httpClient; _artistService = artistService; _albumService = albumService; _metadataProfileService = metadataProfileService; _pluginSettings = pluginSettings; _cache = new CacheService(); } private async Task> CachedSearchAsync(DiscogsMetadataProxySettings settings, string query, Func mapper, string kind = "all", string? artist = null) { string key = $"{kind}:{query}:{artist ?? ""}" + _identifier; DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken, PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; List results = await _cache.FetchAndCacheAsync(key, () => apiService.SearchAsync(new() { Query = query, Artist = artist, Type = kind == "all" ? string.Empty : kind })); return results.Select(r => mapper(r)).Where(x => x != null).ToList()!; } private void UpdateCache(DiscogsMetadataProxySettings settings) { _cache.CacheDirectory = settings.CacheDirectory; _cache.CacheType = (CacheType)settings.RequestCacheType; } public List SearchNewAlbum(DiscogsMetadataProxySettings settings, string title, string artist) { _logger.Debug($"SearchNewAlbum: title '{title}', artist '{artist}'"); UpdateCache(settings); try { List albums = CachedSearchAsync(settings, title, r => { DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken, PageSize = settings.PageSize }; DiscogsRelease? release = apiService.GetReleaseAsync(r.Id).GetAwaiter().GetResult(); return DiscogsMappingHelper.MapAlbumFromRelease(release!); }, "release", artist).GetAwaiter().GetResult(); return albums.GroupBy(a => new { a.CleanTitle, a.AlbumType }).Select(g => g.First()).ToList(); } catch (Exception ex) { _logger.Error($"SearchNewAlbum error: {ex}"); throw; } } public List SearchNewArtist(DiscogsMetadataProxySettings settings, string title) { UpdateCache(settings); return CachedSearchAsync(settings, title, DiscogsMappingHelper.MapArtistFromSearchItem, "artist", null).GetAwaiter().GetResult(); } public List SearchNewEntity(DiscogsMetadataProxySettings settings, string query) { _logger.Debug($"SearchNewEntity invoked: query '{query}'"); UpdateCache(settings); query = SanitizeToUnicode(query); DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken, PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; if (IsDiscogsidQuery(query)) { query = query.Replace("discogs:", "").Replace("discogsid:", ""); string? typeSpecifier = null; if (query.Length > 2 && query[1] == ':') { typeSpecifier = query[0].ToString().ToLowerInvariant(); query = query[2..]; } if (int.TryParse(query, out int discogsId)) { List results = []; if (typeSpecifier == "a") { results.Add(_cache.FetchAndCacheAsync($"artist:{discogsId}", () => apiService.GetArtistAsync(discogsId)!) .ContinueWith(t => t.GetAwaiter().GetResult() == null ? null : (object)DiscogsMappingHelper.MapArtistFromDiscogsArtist(t.GetAwaiter().GetResult())).GetAwaiter().GetResult()); } else if (typeSpecifier == "r") { results.Add(_cache.FetchAndCacheAsync($"release:{discogsId}", () => apiService.GetReleaseAsync(discogsId)!) .ContinueWith(t => t.GetAwaiter().GetResult() == null ? null : (object)DiscogsMappingHelper.MapAlbumFromRelease(t.GetAwaiter().GetResult())) .GetAwaiter().GetResult()); } else if (typeSpecifier == "m") { results.Add(_cache.FetchAndCacheAsync($"master:{discogsId}", () => apiService.GetMasterReleaseAsync(discogsId)!) .ContinueWith(t => t.GetAwaiter().GetResult() == null ? null : (object)DiscogsMappingHelper.MapAlbumFromMasterRelease(t.GetAwaiter().GetResult())) .GetAwaiter().GetResult()); } return results.Where(x => x != null).ToList()!; } } return CachedSearchAsync(settings, query, item => { return item.Type?.ToLowerInvariant() switch { "artist" => _cache.FetchAndCacheAsync($"artist:{item.Id}", () => apiService.GetArtistAsync(item.Id)!) .ContinueWith(t => (object)DiscogsMappingHelper.MapArtistFromDiscogsArtist(t.GetAwaiter().GetResult())).GetAwaiter().GetResult(), "release" => _cache.FetchAndCacheAsync($"release:{item.Id}", () => apiService.GetReleaseAsync(item.Id)!) .ContinueWith(t => { Album album = DiscogsMappingHelper.MapAlbumFromRelease(t.GetAwaiter().GetResult()); album.Artist = DiscogsMappingHelper.MapArtistFromSearchItem(item); return album; }).GetAwaiter().GetResult(), "master" => _cache.FetchAndCacheAsync($"master:{item.Id}", () => apiService.GetMasterReleaseAsync(item.Id)!) .ContinueWith(t => { Album album = DiscogsMappingHelper.MapAlbumFromMasterRelease(t.GetAwaiter().GetResult()); album.Artist = DiscogsMappingHelper.MapArtistFromSearchItem(item); return album; }).GetAwaiter().GetResult(), _ => null }; }).GetAwaiter().GetResult(); } public async Task>> GetAlbumInfoAsync(DiscogsMetadataProxySettings settings, string foreignAlbumId) { _logger.Debug($"Starting GetAlbumInfoAsync for AlbumId: {foreignAlbumId}"); UpdateCache(settings); Album? existingAlbum = _albumService.FindById(foreignAlbumId); bool useMaster = foreignAlbumId.StartsWith('m'); _logger.Trace($"Using {(useMaster ? "master" : "release")} details for AlbumId: {foreignAlbumId}"); DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken }; (Album mappedAlbum, object releaseForTracks) = useMaster ? await GetMasterReleaseDetailsAsync(foreignAlbumId, apiService) : await GetReleaseDetailsAsync(foreignAlbumId, apiService); DiscogsArtist? discogsArtist = await GetPrimaryArtistAsync(foreignAlbumId, useMaster, existingAlbum); Artist existingArtist = (existingAlbum?.Artist?.Value ?? (discogsArtist != null ? DiscogsMappingHelper.MapArtistFromDiscogsArtist(discogsArtist) : null)) ?? throw new ModelNotFoundException(typeof(Artist), 0); _logger.Trace($"Processed artist information for ArtistId: {existingArtist.ForeignArtistId}"); existingArtist.Albums ??= new LazyLoaded>([]); mappedAlbum.Artist = existingArtist; mappedAlbum.ArtistMetadata = existingArtist.Metadata; mappedAlbum.ArtistMetadataId = existingArtist.ArtistMetadataId; Album finalAlbum = DiscogsMappingHelper.MergeAlbums(existingAlbum!, mappedAlbum); AlbumRelease albumRelease = finalAlbum.AlbumReleases.Value[0]; List tracks = DiscogsMappingHelper.MapTracks(releaseForTracks, finalAlbum, albumRelease); _logger.Trace($"Mapped {tracks.Count} tracks for AlbumId: {foreignAlbumId}"); albumRelease.TrackCount = tracks.Count; albumRelease.Duration = tracks.Sum(x => x.Duration); albumRelease.Monitored = tracks.Count > 0; albumRelease.Tracks = tracks; _logger.Trace($"Completed processing for AlbumId: {foreignAlbumId}. Total Tracks: {tracks.Count}"); return new Tuple>(existingArtist.ForeignArtistId, finalAlbum, [existingArtist.Metadata.Value]); } private async Task<(Album, object)> GetMasterReleaseDetailsAsync(string id, DiscogsApiService apiService) { string masterKey = $"master:{id}" + _identifier; DiscogsMasterRelease? master = await _cache.FetchAndCacheAsync(masterKey, () => apiService.GetMasterReleaseAsync(int.Parse(RemoveIdentifier(id[1..])))!); return (DiscogsMappingHelper.MapAlbumFromMasterRelease(master!), master)!; } private async Task<(Album, object)> GetReleaseDetailsAsync(string id, DiscogsApiService apiService) { string releaseKey = $"release:{id}" + _identifier; DiscogsRelease? release = await _cache.FetchAndCacheAsync(releaseKey, () => apiService.GetReleaseAsync(int.Parse(RemoveIdentifier(id[1..])))!); return (DiscogsMappingHelper.MapAlbumFromRelease(release!), release)!; } private async Task GetPrimaryArtistAsync(string id, bool useMaster, Album? existingAlbum) { string key = (useMaster ? $"master:{id}" : $"release:{id}") + _identifier; object? release = useMaster ? await _cache.FetchAndCacheAsync(key, () => Task.FromResult(null!)) : await _cache.FetchAndCacheAsync(key, () => Task.FromResult(null!)); IEnumerable artists = (IEnumerable)(release as dynamic)?.Artists! ?? []; return artists.FirstOrDefault(x => existingAlbum == null || Fuzz.Ratio(x.Name, existingAlbum.Artist?.Value.Name) > 80); } public async Task GetArtistInfoAsync(DiscogsMetadataProxySettings settings, string foreignArtistId, int metadataProfileId) { _logger.Debug($"Fetching artist info for ID: {foreignArtistId}."); UpdateCache(settings); string artistCacheKey = $"artist:{foreignArtistId}" + _identifier; DiscogsArtist? artist = await _cache.FetchAndCacheAsync(artistCacheKey, () => { DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken }; string cleanId = RemoveIdentifier(foreignArtistId); if (cleanId.Length > 1 && !char.IsDigit(cleanId[0])) cleanId = cleanId[1..]; return apiService.GetArtistAsync(int.Parse(cleanId))!; }); Artist? existingArtist = _artistService.FindById(foreignArtistId); existingArtist ??= DiscogsMappingHelper.MapArtistFromDiscogsArtist(artist!); existingArtist.Albums = AlbumMapper.FilterAlbums(await FetchAlbumsForArtistAsync(settings, existingArtist, artist!.Id), metadataProfileId, _metadataProfileService); existingArtist.MetadataProfileId = metadataProfileId; _logger.Trace($"Processed artist: {artist.Name} (ID: {existingArtist.ForeignArtistId})."); return existingArtist; } private async Task> FetchAlbumsForArtistAsync(DiscogsMetadataProxySettings settings, Artist artist, int foreignArtistId) { _logger.Debug($"Fetching albums for artist ID: {foreignArtistId}."); string key = $"artist-albums:{foreignArtistId}" + _identifier; List artistReleases = await _cache.FetchAndCacheAsync(key, () => { DiscogsApiService apiService = new(_httpClient, settings.UserAgent) { AuthToken = settings.AuthToken }; return apiService.GetArtistReleasesAsync(foreignArtistId, null, 70); }); List albums = []; foreach (DiscogsArtistRelease release in artistReleases) { if (release == null || release.Role != "Main") continue; Album album = DiscogsMappingHelper.MapAlbumFromArtistRelease(release); album.Artist = artist; album.ArtistMetadata = artist.Metadata; albums.Add(album); } _logger.Trace($"Fetched {albums.Count} albums for artist ID: {foreignArtistId}."); return albums; } public bool IsDiscogsidQuery(string? query) => query?.StartsWith("discogs:", StringComparison.OrdinalIgnoreCase) == true || query?.StartsWith("discogsid:", StringComparison.OrdinalIgnoreCase) == true; private static string SanitizeToUnicode(string input) => string.IsNullOrEmpty(input) ? input : new string(input.Where(c => c <= 0xFFFF).ToArray()); private static string RemoveIdentifier(string input) => input.EndsWith(_identifier, StringComparison.OrdinalIgnoreCase) ? input.Remove(input.Length - _identifier.Length) : input; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/DiscogsRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public record DiscogsArtist( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("profile")] string? Profile, [property: JsonPropertyName("releases_url")] string? ReleasesUrl, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("urls")] List? Urls, [property: JsonPropertyName("data_quality")] string? DataQuality, [property: JsonPropertyName("namevariations")] List? NameVariations, [property: JsonPropertyName("images")] List? Images, [property: JsonPropertyName("members")] List? Members, [property: JsonPropertyName("join")] string? Join, [property: JsonPropertyName("role")] string? Role, [property: JsonPropertyName("tracks")] string? Tracks ) : MappingAgent; public record DiscogsArtistRelease( [property: JsonPropertyName("artist")] string? Artist, [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("main_release")] int? MainRelease, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("role")] string? Role, [property: JsonPropertyName("thumb")] string? Thumb, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("type")] string? Type, [property: JsonPropertyName("year")] int Year, [property: JsonPropertyName("format")] string? Format, [property: JsonPropertyName("label")] string? Label, [property: JsonPropertyName("status")] string? Status ) : MappingAgent; public record DiscogsCommunityInfo( [property: JsonPropertyName("contributors")] List? Contributors, [property: JsonPropertyName("data_quality")] string? DataQuality, [property: JsonPropertyName("have")] int? Have, [property: JsonPropertyName("rating")] DiscogsRating? Rating, [property: JsonPropertyName("status")] string? Status, [property: JsonPropertyName("submitter")] DiscogsContributor? Submitter, [property: JsonPropertyName("want")] int? Want ); public record DiscogsCompany( [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("entity_type")] string? EntityType, [property: JsonPropertyName("entity_type_name")] string? EntityTypeName, [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("resource_url")] string? ResourceUrl ); public record DiscogsContributor( [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("username")] string? Username ); public record DiscogsFormat( [property: JsonPropertyName("descriptions")] List? Descriptions, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("qty")] string? Qty ); public record DiscogsImage( [property: JsonPropertyName("height")] int Height, [property: JsonPropertyName("width")] int Width, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("type")] string? Type, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("uri150")] string? Uri150 ); public record DiscogsIdentifier( [property: JsonPropertyName("type")] string? Type, [property: JsonPropertyName("value")] string? Value ); public record DiscogsLabel( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("entity_type")] string? EntityType, [property: JsonPropertyName("entity_type_name")] string? EntityTypeName, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("profile")] string? Profile, [property: JsonPropertyName("contact_info")] string? ContactInfo, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("sublabels")] List? Sublabels, [property: JsonPropertyName("urls")] List? Urls, [property: JsonPropertyName("images")] List? Images, [property: JsonPropertyName("data_quality")] string? DataQuality ) : MappingAgent; public record DiscogsLabelRelease( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("artist")] string? Artist, [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("format")] string? Format, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("status")] string? Status, [property: JsonPropertyName("thumb")] string? Thumb, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("year")] string? Year ) : MappingAgent; public record DiscogsMasterRelease( [property: JsonPropertyName("styles")] List? Styles, [property: JsonPropertyName("genres")] List? Genres, [property: JsonPropertyName("videos")] List? Videos, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("main_release")] int MainRelease, [property: JsonPropertyName("main_release_url")] string? MainReleaseUrl, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("artists")] List? Artists, [property: JsonPropertyName("versions_url")] string? VersionsUrl, [property: JsonPropertyName("year")] int Year, [property: JsonPropertyName("images")] List? Images, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("tracklist")] List? Tracklist, [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("num_for_sale")] int NumForSale, [property: JsonPropertyName("lowest_price")] decimal? LowestPrice, [property: JsonPropertyName("data_quality")] string? DataQuality ) : MappingAgent; public record DiscogsMasterReleaseVersion( [property: JsonPropertyName("status")] string? Status, [property: JsonPropertyName("stats")] DiscogsStats? Stats, [property: JsonPropertyName("thumb")] string? Thumb, [property: JsonPropertyName("format")] string? Format, [property: JsonPropertyName("country")] string? Country, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("label")] string? Label, [property: JsonPropertyName("released")] string? Released, [property: JsonPropertyName("major_formats")] List? MajorFormats, [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("id")] int Id ) : MappingAgent; public record DiscogsMember( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("active")] bool Active, [property: JsonPropertyName("resource_url")] string? ResourceUrl ); public record DiscogsRating( [property: JsonPropertyName("average")] decimal Average, [property: JsonPropertyName("count")] int Count ); public record DiscogsRelease( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("artists")] List? Artists, [property: JsonPropertyName("data_quality")] string? DataQuality, [property: JsonPropertyName("thumb")] string? Thumb, [property: JsonPropertyName("community")] DiscogsCommunityInfo? Community, [property: JsonPropertyName("companies")] List? Companies, [property: JsonPropertyName("country")] string? Country, [property: JsonPropertyName("date_added")] DateTime? DateAdded, [property: JsonPropertyName("date_changed")] DateTime? DateChanged, [property: JsonPropertyName("estimated_weight")] int? EstimatedWeight, [property: JsonPropertyName("extraartists")] List? ExtraArtists, [property: JsonPropertyName("format_quantity")] int? FormatQuantity, [property: JsonPropertyName("formats")] List? Formats, [property: JsonPropertyName("genres")] List? Genres, [property: JsonPropertyName("identifiers")] List? Identifiers, [property: JsonPropertyName("images")] List? Images, [property: JsonPropertyName("labels")] List? Labels, [property: JsonPropertyName("lowest_price")] decimal? LowestPrice, [property: JsonPropertyName("master_id")] int? MasterId, [property: JsonPropertyName("master_url")] string? MasterUrl, [property: JsonPropertyName("notes")] string? Notes, [property: JsonPropertyName("num_for_sale")] int? NumForSale, [property: JsonPropertyName("released")] string? Released, [property: JsonPropertyName("released_formatted")] string? ReleasedFormatted, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("series")] List? Series, [property: JsonPropertyName("status")] string? Status, [property: JsonPropertyName("styles")] List? Styles, [property: JsonPropertyName("tracklist")] List? Tracklist, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("videos")] List? Videos, [property: JsonPropertyName("year")] int? Year ) : MappingAgent; public record DiscogsSearchItem( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("country")] string? Country, [property: JsonPropertyName("format")] List? Format, [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("type")] string? Type, [property: JsonPropertyName("style")] List? Style, [property: JsonPropertyName("genre")] List? Genre, [property: JsonPropertyName("label")] List? Label, [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("year")] string? Year, [property: JsonPropertyName("thumb")] string? Thumb, [property: JsonPropertyName("community")] DiscogsCommunityInfo? Community ) : MappingAgent; public record DiscogsSeries( [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("catno")] string? Catno, [property: JsonPropertyName("entity_type")] string? EntityType, [property: JsonPropertyName("entity_type_name")] string? EntityTypeName, [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("resource_url")] string? ResourceUrl, [property: JsonPropertyName("thumbnail_url")] string? ThumbnailUrl ) : MappingAgent; public record DiscogsStats( [property: JsonPropertyName("user")] DiscogsUserStats? User, [property: JsonPropertyName("community")] DiscogsCommunityStats? Community ); public record DiscogsUserStats( [property: JsonPropertyName("in_collection")] int InCollection, [property: JsonPropertyName("in_wantlist")] int InWantlist ); public record DiscogsCommunityStats( [property: JsonPropertyName("in_collection")] int InCollection, [property: JsonPropertyName("in_wantlist")] int InWantlist ); public record DiscogsSublabel( [property: JsonPropertyName("id")] int Id, [property: JsonPropertyName("name")] string? Name, [property: JsonPropertyName("resource_url")] string? ResourceUrl ); public record DiscogsTrack( [property: JsonPropertyName("duration")] string? Duration, [property: JsonPropertyName("position")] string? Position, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("type_")] string? Type ); public record DiscogsTrackPosition( [property: JsonPropertyName("disc_number")] int DiscNumber, [property: JsonPropertyName("track_number")] int TrackNumber ); public record DiscogsVideo( [property: JsonPropertyName("uri")] string? Uri, [property: JsonPropertyName("title")] string? Title, [property: JsonPropertyName("description")] string? Description, [property: JsonPropertyName("duration")] int Duration, [property: JsonPropertyName("embed")] bool Embed ); public record DiscogsSearchParameter( string? Query = null, string? Type = null, string? Title = null, string? ReleaseTitle = null, string? Credit = null, string? Artist = null, string? Anv = null, string? Label = null, string? Genre = null, string? Style = null, string? Country = null, string? Year = null, string? Format = null, string? Catno = null, string? Barcode = null, string? Track = null, string? Submitter = null, string? Contributor = null) { private static readonly Dictionary KeyMappings = new() { { nameof(Query), "q" }, { nameof(Type), "type" }, { nameof(Title), "title" }, { nameof(ReleaseTitle), "release_title" }, { nameof(Credit), "credit" }, { nameof(Artist), "artist" }, { nameof(Anv), "anv" }, { nameof(Label), "label" }, { nameof(Genre), "genre" }, { nameof(Style), "style" }, { nameof(Country), "country" }, { nameof(Year), "year" }, { nameof(Format), "format" }, { nameof(Catno), "catno" }, { nameof(Barcode), "barcode" }, { nameof(Track), "track" }, { nameof(Submitter), "submitter" }, { nameof(Contributor), "contributor" } }; public Dictionary ToDictionary() => GetType() .GetProperties() .Where(prop => prop.PropertyType == typeof(string)) .Select(prop => (Key: KeyMappings[prop.Name], Value: (string?)prop.GetValue(this))) .Where(pair => !string.IsNullOrEmpty(pair.Value)) .ToDictionary(pair => pair.Key, pair => pair.Value!); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Discogs/IDiscogsProxy.cs ================================================ using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Discogs { public interface IDiscogsProxy { List SearchNewAlbum(DiscogsMetadataProxySettings settings, string title, string artist); List SearchNewArtist(DiscogsMetadataProxySettings settings, string title); List SearchNewEntity(DiscogsMetadataProxySettings settings, string title); Task>> GetAlbumInfoAsync(DiscogsMetadataProxySettings settings, string foreignAlbumId); Task GetArtistInfoAsync(DiscogsMetadataProxySettings settings, string lidarrId, int metadataProfileId); bool IsDiscogsidQuery(string? artistName); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/ILastfmProxy.cs ================================================ using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public interface ILastfmProxy { List SearchNewAlbum(LastfmMetadataProxySettings settings, string title, string artist); List SearchNewArtist(LastfmMetadataProxySettings settings, string title); List SearchNewEntity(LastfmMetadataProxySettings settings, string query); Task>> GetAlbumInfoAsync(LastfmMetadataProxySettings settings, string foreignAlbumId); Task GetArtistInfoAsync(LastfmMetadataProxySettings settings, string foreignArtistId, int metadataProfileId); bool IsLastfmIdQuery(string? query); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmApiService.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.Json; using Tubifarry.Core.Model; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public class LastfmApiService { private readonly IHttpClient _httpClient; private readonly Logger _logger; private readonly ICircuitBreaker _circuitBreaker; private readonly string _userAgent; public string? ApiKey { get; set; } public string BaseUrl { get; set; } = "https://ws.audioscrobbler.com/2.0/"; public int MaxRetries { get; set; } = 5; public int InitialRetryDelayMs { get; set; } = 1000; public int MaxPageLimit { get; set; } = 5; public int PageSize { get; set; } = 30; private readonly TimeSpan _rateLimit = TimeSpan.FromSeconds(0.25); public LastfmApiService(IHttpClient httpClient, string userAgent) { _userAgent = userAgent; _httpClient = httpClient; _logger = NzbDroneLogger.GetLogger(this); _circuitBreaker = CircuitBreakerFactory.GetBreaker(this); } /// /// Gets artist info by name /// public async Task GetArtistInfoAsync(string artistName) { Dictionary parameters = new() { { "artist", artistName } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("artist.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmArtistInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return MappingAgent.MapAgent(response?.Artist, _userAgent); } catch (Exception ex) { _logger.Error(ex, "Error deserializing artist info response"); return null; } } /// /// Gets artist info by MusicBrainz ID /// public async Task GetArtistInfoByMbidAsync(string mbid) { Dictionary parameters = new() { { "mbid", mbid } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("artist.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmArtistInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return MappingAgent.MapAgent(response?.Artist, _userAgent); } catch (Exception ex) { _logger.Error(ex, "Error deserializing artist info response"); return null; } } /// /// Searches for artists by name with pagination /// public async Task?> SearchArtistsAsync(string query, int? maxPages = null, int? itemsPerPage = null) => (await FetchPaginatedResultsAsync(BuildRequest("artist.search", new() { { "artist", query } }), maxPages ?? MaxPageLimit, itemsPerPage ?? PageSize))?.Select(x => MappingAgent.MapAgent(x, _userAgent)!).ToList(); /// /// Gets the albums for an artist on Last.fm, ordered by popularity with pagination /// public async Task?> GetTopAlbumsAsync(string? artistName = null, string? mbid = null, int? maxPages = null, int? itemsPerPage = null, bool autoCorrect = false) { if (string.IsNullOrEmpty(artistName) && string.IsNullOrEmpty(mbid)) throw new ArgumentException("Either artistName or mbid must be provided"); Dictionary parameters = []; if (!string.IsNullOrEmpty(artistName)) parameters.Add("artist", artistName); if (!string.IsNullOrEmpty(mbid)) parameters.Add("mbid", mbid); if (autoCorrect) parameters.Add("autocorrect", "1"); List? topAlbums = await FetchPaginatedResultsAsync( BuildRequest("artist.gettopalbums", parameters), maxPages ?? MaxPageLimit, itemsPerPage ?? PageSize); return topAlbums?.Select(x => MappingAgent.MapAgent(x, _userAgent)!).ToList(); } /// /// Gets album info by name and artist /// public async Task GetAlbumInfoAsync(string artistName, string albumName) { Dictionary parameters = new() { { "artist", artistName }, { "album", albumName } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("album.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmAlbumInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return MappingAgent.MapAgent(response?.Album, _userAgent); } catch (Exception ex) { _logger.Error(ex, "Error deserializing album info response"); return null; } } /// /// Gets album info by MusicBrainz ID /// public async Task GetAlbumInfoByMbidAsync(string mbid) { Dictionary parameters = new() { { "mbid", mbid } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("album.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmAlbumInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return MappingAgent.MapAgent(response?.Album, _userAgent); } catch (Exception ex) { _logger.Error(ex, "Error deserializing album info response"); return null; } } /// /// Searches for albums by name with pagination /// public async Task?> SearchAlbumsAsync(string query, int? maxPages = null, int? itemsPerPage = null) => (await FetchPaginatedResultsAsync(BuildRequest("album.search", new() { { "album", query } }), maxPages ?? MaxPageLimit, itemsPerPage ?? PageSize))?.Select(x => MappingAgent.MapAgent(x, _userAgent)!).ToList(); /// /// Gets track info by name and artist /// public async Task GetTrackInfoAsync(string artistName, string trackName) { Dictionary parameters = new() { { "artist", artistName }, { "track", trackName } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("track.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmTrackInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return response?.Track; } catch (Exception ex) { _logger.Error(ex, "Error deserializing track info response"); return null; } } /// /// Gets track info by MusicBrainz ID /// public async Task GetTrackInfoByMbidAsync(string mbid) { Dictionary parameters = new() { { "mbid", mbid } }; JsonElement json = await ExecuteRequestWithRetryAsync(BuildRequest("track.getinfo", parameters)); if (json.ValueKind == JsonValueKind.Undefined) return null; try { LastfmTrackInfoResponse? response = JsonSerializer.Deserialize(json.GetRawText()); return response?.Track; } catch (Exception ex) { _logger.Error(ex, "Error deserializing track info response"); return null; } } /// /// Searches for tracks by name with pagination /// public async Task?> SearchTracksAsync(string query, int? maxPages = null, int? itemsPerPage = null) => await FetchPaginatedResultsAsync(BuildRequest("track.search", new() { { "track", query } }), maxPages ?? MaxPageLimit, itemsPerPage ?? PageSize); /// /// Generic method for fetching paginated results from Last.fm API. /// /// ///TODO: Needs cleaning private async Task?> FetchPaginatedResultsAsync(HttpRequestBuilder requestBuilder, int maxPages, int itemsPerPage) { List results = []; int page = 1; bool hasMorePages = true; while (hasMorePages && page <= maxPages) { HttpRequestBuilder pagedRequest = requestBuilder .AddQueryParam("page", page.ToString(), true) .AddQueryParam("limit", itemsPerPage.ToString(), true); JsonElement response = await ExecuteRequestWithRetryAsync(pagedRequest); if (response.ValueKind == JsonValueKind.Undefined) break; _logger.Trace(response.GetRawText()); try { using JsonDocument doc = JsonDocument.Parse(response.GetRawText()); JsonElement root = doc.RootElement; JsonElement? dataArray = null; foreach (JsonProperty property in root.EnumerateObject()) { if (property.Value.ValueKind != JsonValueKind.Object) continue; // Check each property of this object for arrays foreach (JsonProperty subProperty in property.Value.EnumerateObject()) { // If we find an array, assume it's our data if (subProperty.Value.ValueKind == JsonValueKind.Array) { dataArray = subProperty.Value; break; } // Check one level deeper (common for search results) if (subProperty.Value.ValueKind == JsonValueKind.Object) { foreach (JsonProperty subSubProperty in subProperty.Value.EnumerateObject()) { if (subSubProperty.Value.ValueKind == JsonValueKind.Array) { dataArray = subSubProperty.Value; break; } } if (dataArray.HasValue) break; } } if (dataArray.HasValue) break; } // If we found an array, deserialize and add to results if (dataArray.HasValue) { List? pageItems = JsonSerializer.Deserialize>(dataArray.Value.GetRawText()); if (pageItems?.Count > 0) { results.AddRange(pageItems); } else { hasMorePages = false; } } else { hasMorePages = false; } int totalPages = maxPages; // Try to find pagination in common locations foreach (JsonProperty property in root.EnumerateObject()) { // Skip if not an object if (property.Value.ValueKind != JsonValueKind.Object) continue; // Look for @attr inside this object if (property.Value.TryGetProperty("@attr", out JsonElement attrElement)) { // Try to get totalPages directly if (attrElement.TryGetProperty("totalPages", out JsonElement totalPagesElement) && totalPagesElement.ValueKind == JsonValueKind.String && int.TryParse(totalPagesElement.GetString(), out int parsedPages)) { totalPages = Math.Min(parsedPages, maxPages); break; } // If no totalPages, try calculating from total and perPage if (attrElement.TryGetProperty("total", out JsonElement totalElement) && attrElement.TryGetProperty("perPage", out JsonElement perPageElement) && totalElement.ValueKind == JsonValueKind.String && perPageElement.ValueKind == JsonValueKind.String) { if (int.TryParse(totalElement.GetString(), out int total) && int.TryParse(perPageElement.GetString(), out int perPage) && perPage > 0) { totalPages = Math.Min((int)Math.Ceiling((double)total / perPage), maxPages); break; } } } // Try OpenSearch format (used in search results) if (property.Name == "results") { if (property.Value.TryGetProperty("opensearch:totalResults", out JsonElement totalResults) && property.Value.TryGetProperty("opensearch:itemsPerPage", out JsonElement itemsPerPageElement) && totalResults.ValueKind == JsonValueKind.String && itemsPerPageElement.ValueKind == JsonValueKind.String) { if (int.TryParse(totalResults.GetString(), out int total) && int.TryParse(itemsPerPageElement.GetString(), out int perPage) && perPage > 0) { totalPages = Math.Min((int)Math.Ceiling((double)total / perPage), maxPages); break; } } } } hasMorePages = hasMorePages && page < totalPages; } catch (Exception ex) { _logger.Error(ex, $"Error processing response for page {page}"); break; } page++; } _logger.Trace($"Fetched {results.Count} results across {page - 1} pages."); return results; } /// /// Constructs an HTTP request with the Last.fm API parameters /// private HttpRequestBuilder BuildRequest(string method, Dictionary? parameters = null) { HttpRequestBuilder builder = new(BaseUrl); builder.Headers.Add("User-Agent", _userAgent); builder.AddQueryParam("method", method); builder.AddQueryParam("api_key", ApiKey); builder.AddQueryParam("format", "json"); if (parameters != null) { foreach (KeyValuePair param in parameters) builder.AddQueryParam(param.Key, param.Value); } builder.AllowAutoRedirect = true; builder.SuppressHttpError = true; builder.WithRateLimit(_rateLimit.TotalSeconds); _logger.Trace($"Building request for method: {method}"); return builder; } /// /// Executes an HTTP request with retry logic /// private async Task ExecuteRequestWithRetryAsync(HttpRequestBuilder requestBuilder, int retryCount = 0) { try { if (_circuitBreaker.IsOpen) { _logger.Warn("Circuit breaker is open, skipping request to Last.fm API"); return default; } HttpRequest request = requestBuilder.Build(); HttpResponse response = await _httpClient.GetAsync(request); if (response.StatusCode == HttpStatusCode.TooManyRequests) { if (retryCount >= MaxRetries) { _logger.Warn("Max retries reached due to rate limiting."); _circuitBreaker.RecordFailure(); return default; } int delayMs = InitialRetryDelayMs * (int)Math.Pow(2, retryCount); _logger.Warn($"Rate limit exceeded. Retrying in {delayMs}ms..."); await Task.Delay(delayMs); return await ExecuteRequestWithRetryAsync(requestBuilder, retryCount + 1); } if (response.StatusCode != HttpStatusCode.OK) { HandleErrorResponse(response); _circuitBreaker.RecordFailure(); return default; } _logger.Trace(response.Content); using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); _circuitBreaker.RecordSuccess(); return jsonDoc.RootElement.Clone(); } catch (HttpException ex) { _logger.Warn($"API Error: {ex.Message}"); _circuitBreaker.RecordFailure(); return default; } } /// /// Logs error responses from the API /// private void HandleErrorResponse(HttpResponse response) { try { using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); JsonElement root = jsonDoc.RootElement; if (root.TryGetProperty("error", out JsonElement errorCode) && root.TryGetProperty("message", out JsonElement errorMessage)) _logger.Warn($"Last.fm API Error {errorCode.GetInt32()}: {errorMessage.GetString()}"); else _logger.Warn($"API Error: {response.StatusCode}"); } catch (Exception ex) { _logger.Error(ex, $"Failed to parse API error response. Status Code: {response.StatusCode}"); } } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmImageScraper.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using System.Net; using System.Text.RegularExpressions; using Tubifarry.Core.Model; using Tubifarry.Core.Replacements; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { /// /// Simple scraper that extracts image IDs from Last.fm's artist page /// and constructs direct URLs to high-quality images /// public partial class LastfmImageScraper { private readonly IHttpClient _httpClient; private readonly Logger _logger; private readonly ICircuitBreaker _circuitBreaker; private readonly string _userAgent; private readonly CacheService _cache; public LastfmImageScraper(IHttpClient httpClient, string userAgent, CacheService cache) { _httpClient = httpClient; _userAgent = userAgent; _logger = NzbDroneLogger.GetLogger(this); _circuitBreaker = CircuitBreakerFactory.GetBreaker(this); _cache = cache; } /// /// Gets high-quality images for an artist by extracting image IDs from their Last.fm page /// /// The name of the artist /// A list of direct image URLs public async Task> GetArtistImagesAsync(string artistName) { if (string.IsNullOrWhiteSpace(artistName)) { _logger.Warn("Cannot fetch artist images: Artist name is empty"); return []; } string cacheKey = $"lastfm_artist_images_{artistName.ToLowerInvariant().Replace(" ", "_")}"; try { List? cachedResult = await _cache.GetAsync>(cacheKey); if (cachedResult != null) { _logger.Trace($"Cache hit for {artistName}"); return cachedResult; } List fetchedResult = await FetchArtistImagesAsync(artistName); if (fetchedResult.Count > 0) { await _cache.SetAsync(cacheKey, fetchedResult); _logger.Trace($"Cached {fetchedResult.Count} images for artist {artistName}"); } return fetchedResult; } catch (Exception ex) { _logger.Error(ex, $"Error in cache operation for {artistName}"); return []; } } /// /// Fetches artist images from Last.fm /// /// The name of the artist /// A list of direct image URLs private async Task> FetchArtistImagesAsync(string artistName) { if (_circuitBreaker.IsOpen) { _logger.Warn("Circuit breaker is open, skipping request to Last.fm website"); return []; } try { string safeArtistName = artistName.Replace(" ", "+").Replace("&", "%26"); string url = $"https://www.last.fm/music/{WebUtility.UrlEncode(safeArtistName)}/+images"; _logger.Trace($"Fetching artist images from: {url}"); HttpRequest request = new(url); request.Headers.Add("User-Agent", _userAgent); HttpResponse response = await _httpClient.GetAsync(request); if (response.StatusCode != HttpStatusCode.OK) { _logger.Warn($"Failed to get artist images. Status: {response.StatusCode}"); _circuitBreaker.RecordFailure(); return []; } _circuitBreaker.RecordSuccess(); List imageUrls = ExtractImageUrls(response.Content); _logger.Trace($"Found {imageUrls.Count} high-quality images for artist {artistName}"); return imageUrls; } catch (Exception ex) { _logger.Error(ex, $"Error fetching artist images for {artistName}"); _circuitBreaker.RecordFailure(); return []; } } /// /// Extracts image IDs from the HTML and constructs direct image URLs /// /// HTML content from the artist's images page /// List of full-size image URLs private List ExtractImageUrls(string html) { List imageUrls = []; Match ulMatch = UlContainerRegex().Match(html); if (ulMatch.Success && ulMatch.Groups.Count > 1) { string ulContent = ulMatch.Groups[1].Value; _logger.Trace("Found image list container"); MatchCollection idMatches = ImageIdRegex().Matches(ulContent); _logger.Trace($"Found {idMatches.Count} image IDs in container"); foreach (Match idMatch in idMatches) { if (idMatch.Success && idMatch.Groups.Count > 1) { string imageId = idMatch.Groups[1].Value; if (!string.IsNullOrEmpty(imageId)) { string imageUrl = $"https://lastfm.freetls.fastly.net/i/u/{imageId}.jpg?{FlexibleHttpDispatcher.UA_PARAM}={_userAgent}"; _logger.Trace($"Found image: {imageUrl}"); imageUrls.Add(imageUrl); } } } } else { _logger.Warn("Could not find the image list container in the HTML"); } return imageUrls; } [GeneratedRegex(@"(.+?)", RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex UlContainerRegex(); [GeneratedRegex(@"href=""[^""]*\/\+images\/([0-9a-f]{32})""", RegexOptions.Compiled)] private static partial Regex ImageIdRegex(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmMappingHelper.cs ================================================ using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaCover; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using Tubifarry.Core.Replacements; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public static class LastfmMappingHelper { private const string _identifier = "@lastfm"; /// /// Maps a Last.fm album to a Lidarr album model. /// public static Album MapAlbumFromLastfmAlbum(LastfmAlbum lastfmAlbum, Artist? artist = null) { Album album = new() { ForeignAlbumId = $"{SanitizeName(lastfmAlbum.ArtistName)}::{SanitizeName(lastfmAlbum.Name)}{_identifier}", Title = lastfmAlbum.Name ?? string.Empty, CleanTitle = lastfmAlbum.Name.CleanArtistName(), Links = [], Genres = lastfmAlbum.Tags?.Tag?.Select(g => g.Name).ToList() ?? [], SecondaryTypes = [], AnyReleaseOk = true, Ratings = ComputeLastfmRating(lastfmAlbum.Listeners, lastfmAlbum.PlayCount) }; if (lastfmAlbum.Wiki != null) { album.Overview = !string.IsNullOrEmpty(lastfmAlbum.Wiki.Summary) ? lastfmAlbum.Wiki.Summary : "Found on Last.fm"; } else { List overviewParts = []; if (lastfmAlbum.PlayCount != 0) overviewParts.Add($"Playcount: {lastfmAlbum.PlayCount}"); if (!string.IsNullOrEmpty(lastfmAlbum.Listeners)) overviewParts.Add($"Listeners: {lastfmAlbum.Listeners}"); album.Overview = overviewParts.Count != 0 ? string.Join(" • ", overviewParts) : "Found on Last.fm"; } album.Links.Add(new Links { Url = lastfmAlbum.Url, Name = "Last.fm" }); AddMusicBrainzLink(album.Links, lastfmAlbum.MBID, false); // Map images album.Images = MapImages(lastfmAlbum.Images, lastfmAlbum.UserAgent, false); AlbumRelease albumRelease = new() { ForeignReleaseId = album.ForeignAlbumId, Title = lastfmAlbum.Name, Media = [ new() { Format = "Digital Media", Name = "Digital Media", Number = 1 } ], Album = album, Status = "Official" }; if (artist != null) { album.Artist = artist; album.ArtistMetadata = artist.Metadata; album.ArtistMetadataId = artist.ArtistMetadataId; } if (lastfmAlbum.Tracks?.Tracks?.Count > 0 && artist != null) { int totalDuration = 0; albumRelease.Tracks = new List(); for (int i = 0; i < lastfmAlbum.Tracks.Tracks.Count; i++) { LastfmTrack lastfmTrack = lastfmAlbum.Tracks.Tracks[i]; Track track = MapTrack(lastfmTrack, album, albumRelease, artist, i + 1); albumRelease.Tracks.Value.Add(track); totalDuration += track.Duration; } albumRelease.Duration = totalDuration; albumRelease.TrackCount = lastfmAlbum.Tracks.Tracks.Count; } else { albumRelease.Tracks = new List(); albumRelease.TrackCount = 0; albumRelease.Duration = 0; } if (albumRelease.TrackCount > 3) { album.AlbumType = "Album"; } else if (albumRelease.TrackCount > 1) { bool isSingle = true; for (int i = 0; i < albumRelease.TrackCount - 1 && isSingle; i++) { for (int j = i + 1; j < albumRelease.TrackCount && isSingle; j++) { if (FuzzySharp.Fuzz.Ratio(albumRelease.Tracks.Value[i].Title, albumRelease.Tracks.Value[j].Title) < 80) { isSingle = false; break; } } } album.AlbumType = isSingle ? "Single" : "Album"; } else if (albumRelease.TrackCount == 1) { album.AlbumType = "Single"; } else { album.AlbumType = "Unknown"; } album.SecondaryTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(album.Title); album.AlbumReleases = new LazyLoaded>([albumRelease]); return album; } /// /// Maps a Last.fm artist to a Lidarr artist model. /// public static Artist MapArtistFromLastfmArtist(LastfmArtist lastfmArtist) { ArtistMetadata metadata = new() { ForeignArtistId = SanitizeName(lastfmArtist.Name) + _identifier, Name = lastfmArtist.Name ?? string.Empty, Links = [ new() { Url = lastfmArtist.Url, Name = "Last.fm" } ], Genres = lastfmArtist.Tags?.Tag?.Select(t => t.Name).ToList() ?? [], Members = [], Aliases = lastfmArtist.Similar?.Artists?.Where(a => !string.IsNullOrEmpty(a.Name)) .Select(a => a.Name).Take(5).ToList() ?? [], Status = ArtistStatusType.Continuing, Type = string.Empty }; if (lastfmArtist.Bio != null) { metadata.Overview = !string.IsNullOrEmpty(lastfmArtist.Bio.Summary) ? lastfmArtist.Bio.Summary : "Found on Last.fm"; } else { List overviewParts = []; if (lastfmArtist.Stats != null) { if (lastfmArtist.Stats.PlayCount != 0) overviewParts.Add($"Playcount: {lastfmArtist.Stats.PlayCount}"); if (!string.IsNullOrEmpty(lastfmArtist.Stats.Listeners)) overviewParts.Add($"Listeners: {lastfmArtist.Stats.Listeners}"); } metadata.Overview = overviewParts.Count != 0 ? string.Join(" • ", overviewParts) : "Found on Last.fm"; } metadata.Ratings = ComputeLastfmRating(lastfmArtist.Stats?.Listeners ?? "0", lastfmArtist.Stats?.PlayCount ?? 0); AddMusicBrainzLink(metadata.Links, lastfmArtist.MBID, true); metadata.Images = MapImages(lastfmArtist.Images, lastfmArtist.UserAgent, true); return new() { ForeignArtistId = metadata.ForeignArtistId, Name = lastfmArtist.Name, SortName = lastfmArtist.Name, CleanName = lastfmArtist.Name.CleanArtistName(), Metadata = new LazyLoaded(metadata) }; } /// /// Maps a Last.fm top album to a Lidarr album model. /// public static Album MapAlbumFromLastfmTopAlbum(LastfmTopAlbum topAlbum, Artist artist) { Album album = new() { ForeignAlbumId = $"{artist.ForeignArtistId.Replace(_identifier, "")}::{SanitizeName(topAlbum.Name)}{_identifier}", Title = topAlbum.Name ?? string.Empty, CleanTitle = topAlbum.Name.CleanArtistName(), Links = [new() { Url = topAlbum.Url, Name = "Last.fm" }], AlbumType = "Album", Ratings = new(), SecondaryTypes = [], Overview = $"Found on Last.fm • Playcount: {topAlbum.PlayCount}" }; AlbumRelease albumRelease = new() { ForeignReleaseId = $"{SanitizeName(topAlbum.ArtistName)}::{SanitizeName(topAlbum.Name!)}{_identifier}", Title = topAlbum.Name, Album = album, Status = "Official", Tracks = new List(), }; if (artist != null) { album.Artist = artist; album.ArtistMetadata = artist.Metadata; album.ArtistMetadataId = artist.ArtistMetadataId; } album.SecondaryTypes = AlbumMapper.DetermineSecondaryTypesFromTitle(album.Title); album.AlbumReleases = new LazyLoaded>([albumRelease]); return album; } /// /// Maps a Last.fm track to a Lidarr track model. /// public static Track MapTrack(LastfmTrack lastfmTrack, Album album, AlbumRelease albumRelease, Artist artist, int trackPosition) => new() { ForeignRecordingId = $"{SanitizeName(artist.Name)}::{SanitizeName(album.Title)}::{SanitizeName(lastfmTrack.Name)}{_identifier}", ForeignTrackId = $"{SanitizeName(artist.Name)}::{SanitizeName(album.Title)}::{SanitizeName(lastfmTrack.Name)}{_identifier}", Title = lastfmTrack.Name, Duration = (int)TimeSpan.FromSeconds(lastfmTrack.Duration ?? 0).TotalMilliseconds, TrackNumber = trackPosition.ToString(), AbsoluteTrackNumber = trackPosition, Explicit = false, MediumNumber = albumRelease.Media.FirstOrDefault()?.Number ?? 1, Album = album, AlbumId = album.Id, AlbumRelease = albumRelease, AlbumReleaseId = albumRelease.Id, ArtistMetadata = artist?.Metadata, ArtistMetadataId = artist?.ArtistMetadataId ?? 0, Artist = artist, Ratings = new Ratings() }; /// /// Merges album information if an existing album is found. /// public static Album MergeAlbums(Album existingAlbum, Album mappedAlbum) { if (existingAlbum == null) return mappedAlbum; DateTime? existingReleaseDate = existingAlbum.ReleaseDate; List existingImages = [.. existingAlbum.Images]; existingAlbum.UseMetadataFrom(mappedAlbum); if (existingReleaseDate > DateTime.MinValue) existingAlbum.ReleaseDate = existingReleaseDate; if (existingImages.Count > 0) existingAlbum.Images = existingImages; existingAlbum.Artist = mappedAlbum.Artist ?? existingAlbum.Artist; existingAlbum.ArtistMetadata = mappedAlbum.ArtistMetadata ?? existingAlbum.ArtistMetadata; existingAlbum.AlbumReleases = mappedAlbum.AlbumReleases ?? existingAlbum.AlbumReleases; return existingAlbum; } /// /// Maps Last.fm images to MediaCover objects. /// private static List MapImages(List? images, string userAgent, bool isArtist) { return images? .Where(i => !string.IsNullOrEmpty(i.Url)) .Select(i => new MediaCover { Url = i.Url, CoverType = MapCoverType(i.Size + $"{FlexibleHttpDispatcher.UA_PARAM}={userAgent}", isArtist) }) .ToList() ?? []; } /// /// Adds a MusicBrainz link to the links collection if MBID is available. /// private static void AddMusicBrainzLink(List links, string? mbid, bool isArtist) { if (!string.IsNullOrEmpty(mbid)) { links.Add(new Links { Url = $"https://musicbrainz.org/{(isArtist ? "artist" : "release")}/{mbid}", Name = "MusicBrainz" }); } } /// /// Maps Last.fm image sizes to appropriate MediaCoverTypes. /// private static MediaCoverTypes MapCoverType(string size, bool isArtist) { if (isArtist) { return size?.ToLowerInvariant() switch { "mega" or "extralarge" => MediaCoverTypes.Poster, "large" => MediaCoverTypes.Poster, "medium" => MediaCoverTypes.Headshot, "small" => MediaCoverTypes.Logo, _ => MediaCoverTypes.Poster }; } else { return size?.ToLowerInvariant() switch { "mega" or "extralarge" => MediaCoverTypes.Cover, "large" => MediaCoverTypes.Fanart, _ => MediaCoverTypes.Unknown, }; } } /// /// Sanitizes names for use in foreign IDs by replacing characters that break URL routing /// private static string SanitizeName(string name) { if (string.IsNullOrWhiteSpace(name)) return string.Empty; string decoded = Uri.UnescapeDataString(name); return decoded.Replace("%", "-pct-", StringComparison.Ordinal); } /// /// Computes rating information based on Last.fm listeners and playcount data /// public static Ratings ComputeLastfmRating(string listenersStr, int playcount) { if (!int.TryParse(listenersStr, out int listeners) || playcount == 0) return new Ratings { Value = 0m, Votes = 0 }; if (listeners == 0) return new Ratings { Value = 0m, Votes = 0 }; double ratingValue = Math.Log(Math.Max(10, listeners), 10) * 2; decimal rating = Math.Min(10m, Math.Max(0m, (decimal)ratingValue)); rating = Math.Truncate(rating); return new Ratings { Value = rating, Votes = playcount }; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmMetadataProxy.cs ================================================ using NLog; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using System.Text.RegularExpressions; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo))] [ProxyFor(typeof(IProvideAlbumInfo))] [ProxyFor(typeof(ISearchForNewArtist))] [ProxyFor(typeof(ISearchForNewAlbum))] [ProxyFor(typeof(ISearchForNewEntity))] public partial class LastfmMetadataProxy(ILastfmProxy lastfmProxy, Logger logger) : ProxyBase, IMetadata, ISupportMetadataMixing { private readonly ILastfmProxy _lastfmProxy = lastfmProxy; private readonly Logger _logger = logger; public override string Name => "Last.fm"; private LastfmMetadataProxySettings ActiveSettings => Settings ?? LastfmMetadataProxySettings.Instance!; public List SearchForNewAlbum(string title, string artist) => _lastfmProxy.SearchNewAlbum(ActiveSettings, title, artist); public List SearchForNewArtist(string title) => _lastfmProxy.SearchNewArtist(ActiveSettings, title); public List SearchForNewEntity(string title) => _lastfmProxy.SearchNewEntity(ActiveSettings, title); public Tuple> GetAlbumInfo(string foreignAlbumId) => _lastfmProxy.GetAlbumInfoAsync(ActiveSettings, foreignAlbumId).GetAwaiter().GetResult(); public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => _lastfmProxy.GetArtistInfoAsync(ActiveSettings, lidarrId, metadataProfileId).GetAwaiter().GetResult(); public HashSet GetChangedAlbums(DateTime startTime) { _logger.Warn("GetChangedAlbums: Last.fm API does not support change tracking; returning empty set."); return []; } public HashSet GetChangedArtists(DateTime startTime) { _logger.Warn("GetChangedArtists: Last.fm API does not support change tracking; returning empty set."); return []; } public List SearchForNewAlbumByRecordingIds(List recordingIds) { _logger.Warn("SearchNewAlbumByRecordingIds: Last.fm API does not support fingerprint search; returning empty list."); return []; } public MetadataSupportLevel CanHandleSearch(string? albumTitle, string? artistName) { if (_lastfmProxy.IsLastfmIdQuery(albumTitle) || _lastfmProxy.IsLastfmIdQuery(artistName)) return MetadataSupportLevel.Supported; if ((albumTitle != null && FormatRegex().IsMatch(albumTitle)) || (artistName != null && FormatRegex().IsMatch(artistName))) return MetadataSupportLevel.Unsupported; return MetadataSupportLevel.ImplicitSupported; } public MetadataSupportLevel CanHandleId(string id) { if (id.EndsWith("@lastfm")) return MetadataSupportLevel.Supported; else return MetadataSupportLevel.Unsupported; } public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) => MetadataSupportLevel.Unsupported; public MetadataSupportLevel CanHandleChanged() => MetadataSupportLevel.Unsupported; public string? SupportsLink(List links) { if (links == null || links.Count == 0) return null; foreach (Links link in links) { if (string.IsNullOrWhiteSpace(link.Url)) continue; Match match = FormatRegex().Match(link.Url); if (match.Success && match.Groups.Count > 1) { string artistName = match.Groups[1].Value; return $"lastfm:{artistName}"; } } return null; } [GeneratedRegex(@"^\s*\w+:\s*\w+", RegexOptions.Compiled)] private static partial Regex FormatRegex(); [GeneratedRegex(@"last\.fm\/music\/([^\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled, "de-DE")] private static partial Regex LastfmRegex(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public class LastfmMetadataProxySettingsValidator : AbstractValidator { public LastfmMetadataProxySettingsValidator() { #if !DEV_BRANCH RuleFor(x => x) .Must(x => false) .WithMessage("This version of Tubifarry does not support Last.fm for metadata. You would need to switch to develop!"); #endif RuleFor(x => x.ApiKey) .NotEmpty() .WithMessage("A Last.fm API key is required."); // Validate PageNumber must be greater than 0 RuleFor(x => x.PageNumber) .GreaterThan(0) .WithMessage("Page number must be greater than 0."); // Validate PageSize must be greater than 0 RuleFor(x => x.PageSize) .GreaterThan(0) .WithMessage("Page size must be greater than 0."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(4) || DateTime.UtcNow - Tubifarry.LastStarted > TimeSpan.FromDays(5)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // When using Permanent cache, require a valid CacheDirectory RuleFor(x => x.CacheDirectory) .Must((settings, path) => settings.RequestCacheType != (int)CacheType.Permanent || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(4) || DateTime.UtcNow - Tubifarry.LastStarted > TimeSpan.FromDays(5)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // Validate that Warn is checked RuleFor(x => x.UseAtOwnRisk) .Equal(true) .WithMessage("You must acknowledge that this feature is in alpha state by checking the 'Warning' box."); } } public class LastfmMetadataProxySettings : IProviderConfig { private static readonly LastfmMetadataProxySettingsValidator Validator = new(); #if !DEV_BRANCH [FieldDefinition(1, Label = "API Key", Type = FieldType.Textbox, Privacy = PrivacyLevel.ApiKey, HelpText = "Your Last.fm API key", Placeholder = "Enter your API key", Hidden = HiddenType.Hidden)] public string ApiKey { get; set; } = string.Empty; [FieldDefinition(2, Label = "User Agent", Section = MetadataSectionType.Metadata, Type = FieldType.Textbox, HelpText = "Specify a custom User-Agent to identify yourself. A User-Agent helps servers understand the software making the request. Use a unique identifier that includes a name and version. Avoid generic or suspicious-looking User-Agents to prevent blocking.", Placeholder = "Lidarr/1.0.0", Hidden = HiddenType.Hidden)] public string UserAgent { get; set; } = string.Empty; [FieldDefinition(3, Label = "Page Number", Type = FieldType.Number, HelpText = "Page number for pagination", Placeholder = "1", Hidden = HiddenType.Hidden)] public int PageNumber { get; set; } = 1; [FieldDefinition(4, Label = "Page Size", Type = FieldType.Number, HelpText = "Page size for pagination", Placeholder = "30", Hidden = HiddenType.Hidden)] public int PageSize { get; set; } = 3; [FieldDefinition(5, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching", Hidden = HiddenType.Hidden)] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(6, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory to store cached data (only used for Permanent caching)", Hidden = HiddenType.Hidden)] public string CacheDirectory { get; set; } = string.Empty; [FieldDefinition(7, Label = "Warning", Type = FieldType.Checkbox, HelpText = "Use at your own risk this is not ready and is not in beta but in alpha state", Hidden = HiddenType.Hidden)] public bool UseAtOwnRisk { get; set; } #else [FieldDefinition(1, Label = "API Key", Type = FieldType.Textbox, Privacy = PrivacyLevel.ApiKey, HelpText = "Your Last.fm API key", Placeholder = "Enter your API key")] public string ApiKey { get; set; } = string.Empty; [FieldDefinition(2, Label = "User Agent", Section = MetadataSectionType.Metadata, Type = FieldType.Textbox, HelpText = "Specify a custom User-Agent to identify yourself. A User-Agent helps servers understand the software making the request. Use a unique identifier that includes a name and version. Avoid generic or suspicious-looking User-Agents to prevent blocking.", Placeholder = "Lidarr/1.0.0")] public string UserAgent { get; set; } = string.Empty; [FieldDefinition(3, Label = "Page Number", Type = FieldType.Number, HelpText = "Page number for pagination", Placeholder = "1")] public int PageNumber { get; set; } = 1; [FieldDefinition(4, Label = "Page Size", Type = FieldType.Number, HelpText = "Page size for pagination", Placeholder = "30")] public int PageSize { get; set; } = 3; [FieldDefinition(5, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching")] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(6, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory to store cached data (only used for Permanent caching)")] public string CacheDirectory { get; set; } = string.Empty; [FieldDefinition(7, Label = "Warning", Type = FieldType.Checkbox, HelpText = "Use at your own risk this is not ready and is not in beta but in alpha state")] public bool UseAtOwnRisk { get; set; } #endif public LastfmMetadataProxySettings() => Instance = this; public static LastfmMetadataProxySettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmProxy.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaCover; using NzbDrone.Core.MetadataSource.SkyHook; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Metadata; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public class LastfmProxy : ILastfmProxy { private const string _identifier = "@lastfm"; private readonly Logger _logger; private readonly CacheService _cache; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly IHttpClient _httpClient; private readonly IMetadataProfileService _metadataProfileService; public LastfmProxy(Logger logger, IHttpClient httpClient, IArtistService artistService, IAlbumService albumService, IMetadataProfileService metadataProfileService) { _logger = logger; _httpClient = httpClient; _artistService = artistService; _albumService = albumService; _metadataProfileService = metadataProfileService; _cache = new CacheService(); } private void UpdateCache(LastfmMetadataProxySettings settings) { _cache.CacheDirectory = settings.CacheDirectory; _cache.CacheType = (CacheType)settings.RequestCacheType; } public List SearchNewAlbum(LastfmMetadataProxySettings settings, string title, string artist) { _logger.Debug($"SearchNewAlbum: title '{title}', artist '{artist}'"); UpdateCache(settings); try { LastfmApiService apiService = GetApiService(settings); List albums = []; List? lastfmAlbums = _cache.FetchAndCacheAsync($"search:album:{title}:{settings.PageSize}:{settings.PageNumber}", () => apiService.SearchAlbumsAsync(title, settings.PageSize, settings.PageNumber)).GetAwaiter().GetResult(); if (!string.IsNullOrEmpty(artist)) { lastfmAlbums = lastfmAlbums?.Where(a => string.Equals(a.ArtistName, artist, StringComparison.OrdinalIgnoreCase) || a.ArtistName.Contains(artist, StringComparison.OrdinalIgnoreCase)).ToList(); } foreach (LastfmAlbum lastfmAlbum in lastfmAlbums ?? []) { LastfmAlbum? detailedAlbum = _cache.FetchAndCacheAsync($"album:{lastfmAlbum.ArtistName}:{lastfmAlbum.Name}", () => apiService.GetAlbumInfoAsync(lastfmAlbum.ArtistName, lastfmAlbum.Name)).GetAwaiter().GetResult(); if (detailedAlbum != null) albums.Add(LastfmMappingHelper.MapAlbumFromLastfmAlbum(detailedAlbum)); } return albums.GroupBy(a => a.CleanTitle).Select(g => g.First()).ToList(); } catch (Exception ex) { _logger.Error($"SearchNewAlbum error: {ex}"); throw; } } public List SearchNewArtist(LastfmMetadataProxySettings settings, string title) { _logger.Debug($"SearchNewArtist: title '{title}'"); UpdateCache(settings); try { LastfmApiService apiService = GetApiService(settings); List artists = []; List? lastfmArtists = _cache.FetchAndCacheAsync($"search:artist:{title}:{settings.PageSize}:{settings.PageNumber}", () => apiService.SearchArtistsAsync(title, settings.PageSize, settings.PageNumber)).GetAwaiter().GetResult(); foreach (LastfmArtist lastfmArtist in lastfmArtists ?? []) { LastfmArtist? detailedArtist = _cache.FetchAndCacheAsync($"artist:{lastfmArtist.Name}", () => apiService.GetArtistInfoAsync(lastfmArtist.Name)).GetAwaiter().GetResult(); if (detailedArtist != null) artists.Add(LastfmMappingHelper.MapArtistFromLastfmArtist(detailedArtist)); } return artists; } catch (Exception ex) { _logger.Error($"SearchNewArtist error: {ex}"); throw; } } public List SearchNewEntity(LastfmMetadataProxySettings settings, string query) { _logger.Info($"SearchNewEntity invoked: query '{query}'"); UpdateCache(settings); query = SanitizeToUnicode(query); List results = []; try { LastfmApiService apiService = GetApiService(settings); if (IsLastfmIdQuery(query)) { string id = query.Replace("lastfm:", "").Replace("lastfmid:", ""); _logger.Debug($"Processing Last.fm ID query: {id}"); if (Guid.TryParse(id, out _)) { try { results.Add(GetArtistInfoAsync(settings, id, default).GetAwaiter().GetResult()); return results; } catch { try { Tuple> albumResult = GetAlbumInfoAsync(settings, id).GetAwaiter().GetResult(); results.Add(albumResult.Item2); return results; } catch { } } } else if (id.Contains("::")) { try { Tuple> albumResult = GetAlbumInfoAsync(settings, id + _identifier).GetAwaiter().GetResult(); results.Add(albumResult.Item2); return results; } catch { } } else { try { results.Add(GetArtistInfoAsync(settings, id + _identifier, default).GetAwaiter().GetResult()); return results; } catch { } } } _logger.Trace($"Performing general search for: {query}"); List? lastfmArtists = _cache.FetchAndCacheAsync($"search:artist:{query}:{settings.PageSize}:{settings.PageNumber}", () => apiService.SearchArtistsAsync(query, settings.PageSize, settings.PageNumber)).GetAwaiter().GetResult(); foreach (LastfmArtist lastfmArtist in lastfmArtists ?? []) { try { Artist artist = LastfmMappingHelper.MapArtistFromLastfmArtist(lastfmArtist); artist.Albums = new LazyLoaded>([]); results.Add(artist); } catch { } } List? lastfmAlbums = _cache.FetchAndCacheAsync($"search:album:{query}:{settings.PageSize}:{settings.PageNumber}", () => apiService.SearchAlbumsAsync(query, settings.PageSize, settings.PageNumber)).GetAwaiter().GetResult(); foreach (LastfmAlbum lastfmAlbum in lastfmAlbums ?? []) { try { Tuple> albumResult = GetAlbumInfoAsync(settings, $"{lastfmAlbum.ArtistName}::{lastfmAlbum.Name}{_identifier}").GetAwaiter().GetResult(); results.Add(albumResult.Item2); } catch { } } return results; } catch (Exception ex) { _logger.Error($"SearchNewEntity error: {ex}"); throw; } } private async Task EnhanceArtistImagesAsync(LastfmMetadataProxySettings settings, Artist artist) { if (artist?.Metadata?.Value == null || string.IsNullOrEmpty(artist.Name)) return; try { LastfmImageScraper scraper = new(_httpClient, settings.UserAgent, _cache); List imageUrls = await scraper.GetArtistImagesAsync(artist.Name); if (imageUrls == null || imageUrls.Count == 0) return; List newImages = []; for (int i = 0; i < Math.Min(imageUrls.Count, 3); i++) { MediaCoverTypes type = i == 0 ? MediaCoverTypes.Poster : MediaCoverTypes.Fanart; newImages.Add(new MediaCover { Url = imageUrls[i], CoverType = type }); } artist.Metadata.Value.Images = newImages; _logger.Debug($"Enhanced {artist.Name} with {newImages.Count} scraped images"); } catch (Exception ex) { _logger.Warn(ex, $"Failed to enhance artist images for {artist.Name}"); } } public async Task>> GetAlbumInfoAsync(LastfmMetadataProxySettings settings, string foreignAlbumId) { _logger.Info($"Starting GetAlbumInfoAsync for AlbumId: {foreignAlbumId}"); UpdateCache(settings); _logger.Info(foreignAlbumId); try { string albumIdentifier = RemoveIdentifier(foreignAlbumId); LastfmApiService apiService = GetApiService(settings); LastfmAlbum? lastfmAlbum = null; string? artistName = null; string? albumName = null; if (Guid.TryParse(albumIdentifier, out _)) { lastfmAlbum = await _cache.FetchAndCacheAsync( $"album:mbid:{albumIdentifier}", () => apiService.GetAlbumInfoByMbidAsync(albumIdentifier)); } else if (albumIdentifier.Contains("::")) { string[] parts = albumIdentifier.Split(["::"], StringSplitOptions.None); artistName = parts[0].Trim(); albumName = parts.Length > 1 ? parts[1].Trim() : string.Empty; if (!string.IsNullOrEmpty(artistName) && !string.IsNullOrEmpty(albumName)) { lastfmAlbum = await _cache.FetchAndCacheAsync( $"album:{artistName}:{albumName}", () => apiService.GetAlbumInfoAsync(artistName, albumName)); } } else { throw new SkyHookException("Album format not correct"); } if (lastfmAlbum == null) { _logger.Warn($"Album not found on Last.fm: {foreignAlbumId}"); throw new SkyHookException("Album not found on Last.fm"); } LastfmArtist? lastfmArtist = await _cache.FetchAndCacheAsync( $"artist:{lastfmAlbum.ArtistName}", () => apiService.GetArtistInfoAsync(lastfmAlbum.ArtistName)); if (lastfmArtist == null) { _logger.Warn($"Artist not found for album: {foreignAlbumId}"); throw new SkyHookException("Artist not found"); } Artist artist = LastfmMappingHelper.MapArtistFromLastfmArtist(lastfmArtist); Album mappedAlbum = LastfmMappingHelper.MapAlbumFromLastfmAlbum(lastfmAlbum, artist); Album? existingAlbum = _albumService.FindById(foreignAlbumId); if (existingAlbum != null) mappedAlbum = LastfmMappingHelper.MergeAlbums(existingAlbum, mappedAlbum); _logger.Trace($"Completed processing for AlbumId: {foreignAlbumId}"); return new Tuple>(artist.ForeignArtistId, mappedAlbum, [artist.Metadata.Value]); } catch (Exception ex) { _logger.Error($"GetAlbumInfoAsync error: {ex}"); throw; } } public async Task GetArtistInfoAsync(LastfmMetadataProxySettings settings, string foreignArtistId, int metadataProfileId) { _logger.Trace($"Fetching artist info for ID: {foreignArtistId}"); UpdateCache(settings); try { string artistIdentifier = RemoveIdentifier(foreignArtistId); LastfmApiService apiService = GetApiService(settings); LastfmArtist? lastfmArtist = null; if (Guid.TryParse(artistIdentifier, out _)) { lastfmArtist = await _cache.FetchAndCacheAsync( $"artist:mbid:{artistIdentifier}", () => apiService.GetArtistInfoByMbidAsync(artistIdentifier)); } else { lastfmArtist = await _cache.FetchAndCacheAsync( $"artist:{artistIdentifier}", () => apiService.GetArtistInfoAsync(artistIdentifier)); } if (lastfmArtist == null) { _logger.Warn($"Artist not found on Last.fm: {foreignArtistId}"); throw new SkyHookException("Artist not found on Last.fm"); } Artist artist = LastfmMappingHelper.MapArtistFromLastfmArtist(lastfmArtist); await FetchAlbumsForArtistAsync(settings, artist); await EnhanceArtistImagesAsync(settings, artist); artist.Albums = AlbumMapper.FilterAlbums( artist.Albums.Value, metadataProfileId, _metadataProfileService); artist.MetadataProfileId = metadataProfileId; Artist? existingArtist = _artistService.FindById(foreignArtistId); if (existingArtist != null) { // TODO: Merge any existing data artist.Id = existingArtist.Id; artist.Path = existingArtist.Path; artist.Monitored = existingArtist.Monitored; } _logger.Trace($"Processed artist: {artist.Name} (ID: {artist.ForeignArtistId})"); return artist; } catch (Exception ex) { _logger.Error($"GetArtistInfoAsync error: {ex}"); throw; } } /// /// Fetches and maps albums for an artist using the top albums endpoint /// private async Task FetchAlbumsForArtistAsync(LastfmMetadataProxySettings settings, Artist artist) { _logger.Debug($"Fetching top albums for artist: {artist.Name}"); try { LastfmApiService apiService = GetApiService(settings); string artistName = artist.Name; string? mbid = null; LastfmArtist? lastfmArtist = await _cache.FetchAndCacheAsync( $"artist:{RemoveIdentifier(artist.ForeignArtistId)}", () => apiService.GetArtistInfoAsync(artistName)); if (lastfmArtist != null && !string.IsNullOrEmpty(lastfmArtist.MBID)) mbid = lastfmArtist.MBID; string cacheKey = !string.IsNullOrEmpty(mbid) ? $"artist:topalbums:mbid:{mbid}" : $"artist:topalbums:{artistName}"; List? topAlbums = await _cache.FetchAndCacheAsync(cacheKey, () => apiService.GetTopAlbumsAsync(artistName, mbid, 100, null, true)); _logger.Info("Found: " + topAlbums?.Count); List albums = []; foreach (LastfmTopAlbum topAlbum in topAlbums ?? []) { LastfmAlbum? detailedAlbum = await _cache.GetAsync($"album:{topAlbum.ArtistName}::{topAlbum.Name}"); if (detailedAlbum != null) albums.Add(LastfmMappingHelper.MapAlbumFromLastfmAlbum(detailedAlbum, artist)); else albums.Add(LastfmMappingHelper.MapAlbumFromLastfmTopAlbum(topAlbum, artist)); } artist.Albums = new LazyLoaded>(albums.Where(x => x.Title != "(null)").ToList()); _logger.Info($"Fetched {albums.Count} top albums for artist: {artist.Name}"); } catch (Exception ex) { _logger.Error($"Error fetching top albums for artist {artist.Name}: {ex.Message}"); artist.Albums = new LazyLoaded>([]); } } /// /// Gets a configured Last.fm API service /// private LastfmApiService GetApiService(LastfmMetadataProxySettings settings) => new(_httpClient, settings.UserAgent) { ApiKey = settings.ApiKey, PageSize = settings.PageSize, MaxPageLimit = settings.PageNumber }; public bool IsLastfmIdQuery(string? query) => query?.StartsWith("lastfm:", StringComparison.OrdinalIgnoreCase) == true || query?.StartsWith("lastfmid:", StringComparison.OrdinalIgnoreCase) == true; private static string SanitizeToUnicode(string input) => string.IsNullOrEmpty(input) ? input : new string(input.Where(c => c <= 0xFFFF).ToArray()); private static string RemoveIdentifier(string input) => input.EndsWith(_identifier, StringComparison.OrdinalIgnoreCase) ? input.Remove(input.Length - _identifier.Length) : input; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmRecordConverter.cs ================================================ using System.Text.Json; using System.Text.Json.Serialization; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public class LastfmTagsConverter : JsonConverter { public override LastfmTags? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.String) { reader.GetString(); return null; } return JsonSerializer.Deserialize(ref reader, options); } public override void Write(Utf8JsonWriter writer, LastfmTags? value, JsonSerializerOptions options) { if (value == null) writer.WriteStringValue(""); else JsonSerializer.Serialize(writer, value, options); } } public class LastfmNumberConverter : JsonConverter { public override int Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { switch (reader.TokenType) { case JsonTokenType.String: string? stringValue = reader.GetString(); return string.IsNullOrEmpty(stringValue) ? 0 : int.TryParse(stringValue, out int result) ? result : 0; case JsonTokenType.Number: return reader.GetInt32(); case JsonTokenType.Null: return 0; default: throw new JsonException($"Unexpected token type: {reader.TokenType}"); } } public override void Write(Utf8JsonWriter writer, int value, JsonSerializerOptions options) => writer.WriteNumberValue(value); } public class LastfmArtistConverter : JsonConverter { public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.String) { reader.GetString(); } else if (reader.TokenType == JsonTokenType.StartObject) { using JsonDocument doc = JsonDocument.ParseValue(ref reader); if (doc.RootElement.TryGetProperty("name", out JsonElement nameElement)) return nameElement.GetString() ?? ""; return string.Empty; } return string.Empty; } public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => writer.WriteStringValue(value); } public class LastfmTracksConverter : JsonConverter { public override LastfmTracks Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException($"Expected StartObject but got {reader.TokenType}"); List tracks = []; bool foundTrackProperty = false; while (reader.Read()) { if (reader.TokenType == JsonTokenType.EndObject) break; if (reader.TokenType != JsonTokenType.PropertyName) continue; string propertyName = reader.GetString()!; reader.Read(); if (propertyName == "track") { foundTrackProperty = true; if (reader.TokenType == JsonTokenType.StartArray) { tracks = JsonSerializer.Deserialize>(ref reader, options) ?? []; } else if (reader.TokenType == JsonTokenType.StartObject) { LastfmTrack? singleTrack = JsonSerializer.Deserialize(ref reader, options); if (singleTrack != null) tracks.Add(singleTrack); } else if (reader.TokenType == JsonTokenType.Null) { tracks = []; } } else { reader.Skip(); } } if (!foundTrackProperty) tracks = []; return new LastfmTracks(tracks); } public override void Write(Utf8JsonWriter writer, LastfmTracks value, JsonSerializerOptions options) { writer.WriteStartObject(); writer.WritePropertyName("track"); if (value.Tracks.Count == 0) writer.WriteNullValue(); else if (value.Tracks.Count == 1) JsonSerializer.Serialize(writer, value.Tracks[0], options); else JsonSerializer.Serialize(writer, value.Tracks, options); writer.WriteEndObject(); } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Lastfm/LastfmRecords.cs ================================================ using System.Text.Json.Serialization; using Tubifarry.Core.Records; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm { public record LastfmErrorResponse( [property: JsonPropertyName("error")] int Error, [property: JsonPropertyName("message")] string Message ); public record LastfmImage( [property: JsonPropertyName("#text")] string Url, [property: JsonPropertyName("size")] string Size ); public record LastfmStats( [property: JsonPropertyName("listeners")] string Listeners, [property: JsonPropertyName("playcount"), JsonConverter(typeof(LastfmNumberConverter))] int PlayCount ); public record LastfmTag( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("url")] string Url ); public record LastfmTags( [property: JsonPropertyName("tag")] List Tag ); public record LastfmBio( [property: JsonPropertyName("published")] string Published, [property: JsonPropertyName("summary")] string Summary, [property: JsonPropertyName("content")] string Content ); public record LastfmWiki( [property: JsonPropertyName("published")] string Published, [property: JsonPropertyName("summary")] string Summary, [property: JsonPropertyName("content")] string Content ); public record LastfmStreamable( [property: JsonPropertyName("#text")] string Text, [property: JsonPropertyName("fulltrack")] string Fulltrack ); public record LastfmArtist( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("mbid")] string MBID, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("image")] List Images, [property: JsonPropertyName("streamable")] string Streamable, [property: JsonPropertyName("ontour")] string OnTour, [property: JsonPropertyName("stats")] LastfmStats Stats, [property: JsonPropertyName("similar")] LastfmSimilar Similar, [property: JsonPropertyName("tags"), JsonConverter(typeof(LastfmTagsConverter))] LastfmTags? Tags, [property: JsonPropertyName("bio")] LastfmBio Bio ) : MappingAgent; public record LastfmSimilar( [property: JsonPropertyName("artist")] List Artists ); public record LastfmArtistInfoResponse( [property: JsonPropertyName("artist")] LastfmArtist Artist ); public record LastfmTrackArtist( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("mbid")] string MBID, [property: JsonPropertyName("url")] string Url ); public record LastfmTrackAlbum( [property: JsonPropertyName("artist")] string Artist, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("mbid")] string MBID, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("image")] List Images ); public record LastfmTrack( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("mbid")] string MBID, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("duration")] int? Duration, [property: JsonPropertyName("streamable")] LastfmStreamable Streamable, [property: JsonPropertyName("listeners")] string Listeners, [property: JsonPropertyName("playcount"), JsonConverter(typeof(LastfmNumberConverter))] int PlayCount, [property: JsonPropertyName("artist")] LastfmTrackArtist Artist, [property: JsonPropertyName("album")] LastfmTrackAlbum Album, [property: JsonPropertyName("toptags"), JsonConverter(typeof(LastfmTagsConverter))] LastfmTags? Tags, [property: JsonPropertyName("wiki")] LastfmWiki Wiki ); public record LastfmTracks( [property: JsonPropertyName("track")] List Tracks ); public record LastfmTrackMatches( [property: JsonPropertyName("track")] List Tracks ); public record LastfmTrackInfoResponse( [property: JsonPropertyName("track")] LastfmTrack Track ); public record LastfmAlbum( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("artist")] string ArtistName, [property: JsonPropertyName("mbid")] string MBID, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("image")] List Images, [property: JsonPropertyName("listeners")] string Listeners, [property: JsonPropertyName("playcount"), JsonConverter(typeof(LastfmNumberConverter))] int PlayCount, [property: JsonPropertyName("tracks"), JsonConverter(typeof(LastfmTracksConverter))] LastfmTracks Tracks, [property: JsonPropertyName("tags"), JsonConverter(typeof(LastfmTagsConverter))] LastfmTags? Tags, [property: JsonPropertyName("wiki")] LastfmWiki Wiki ) : MappingAgent; public record LastfmAlbumInfoResponse( [property: JsonPropertyName("album")] LastfmAlbum Album ); public record LastfmSearchAttr( [property: JsonPropertyName("for")] string Query, [property: JsonPropertyName("totalResults")] string TotalResults, [property: JsonPropertyName("startPage")] string StartPage, [property: JsonPropertyName("itemsPerPage")] string ItemsPerPage ); public record LastfmTopAlbum( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("playcount"), JsonConverter(typeof(LastfmNumberConverter))] int PlayCount, [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("artist"), JsonConverter(typeof(LastfmArtistConverter))] string ArtistName, [property: JsonPropertyName("image")] List Images ) : MappingAgent; public record LastfmSearchParameter( string? Query = null, string? ArtistName = null, string? AlbumName = null, string? TrackName = null, int? Limit = null, int? Page = null ) { public Dictionary ToDictionary() { Dictionary dict = []; if (!string.IsNullOrEmpty(Query)) dict["q"] = Query; if (!string.IsNullOrEmpty(ArtistName)) dict["artist"] = ArtistName; if (!string.IsNullOrEmpty(AlbumName)) dict["album"] = AlbumName; if (!string.IsNullOrEmpty(TrackName)) dict["track"] = TrackName; if (Limit.HasValue) dict["limit"] = Limit.Value.ToString(); if (Page.HasValue) dict["page"] = Page.Value.ToString(); return dict; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/MetadataProviderWrapper.cs ================================================ using NzbDrone.Common.Http; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider { public class MetadataProviderWrapper(Lazy proxyService) : ProxyWrapperBase(proxyService), IProvideArtistInfo, IProvideAlbumInfo, ISearchForNewArtist, ISearchForNewAlbum, ISearchForNewEntity { // IProvideArtistInfo implementation public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => InvokeProxyMethod( typeof(IProvideArtistInfo), nameof(GetArtistInfo), lidarrId, metadataProfileId); public HashSet GetChangedArtists(DateTime startTime) => InvokeProxyMethod>( typeof(IProvideArtistInfo), nameof(GetChangedArtists), startTime); // IProvideAlbumInfo implementation public Tuple> GetAlbumInfo(string id) => InvokeProxyMethod>>( typeof(IProvideAlbumInfo), nameof(GetAlbumInfo), id); public HashSet GetChangedAlbums(DateTime startTime) => InvokeProxyMethod>( typeof(IProvideAlbumInfo), nameof(GetChangedAlbums), startTime); // ISearchForNewArtist implementation public List SearchForNewArtist(string title) => InvokeProxyMethod>( typeof(ISearchForNewArtist), nameof(SearchForNewArtist), title); // ISearchForNewAlbum implementation public List SearchForNewAlbum(string title, string artist) => InvokeProxyMethod>( typeof(ISearchForNewAlbum), nameof(SearchForNewAlbum), title, artist); public List SearchForNewAlbumByRecordingIds(List recordingIds) => InvokeProxyMethod>( typeof(ISearchForNewAlbum), nameof(SearchForNewAlbumByRecordingIds), recordingIds); // ISearchForNewEntity implementation public List SearchForNewEntity(string title) => InvokeProxyMethod>( typeof(ISearchForNewEntity), nameof(SearchForNewEntity), title); } public class MetadataRequestBuilderWrapper(Lazy proxyService) : ProxyWrapperBase(proxyService), IMetadataRequestBuilder { public IHttpRequestBuilderFactory GetRequestBuilder() => InvokeProxyMethod( typeof(IMetadataRequestBuilder), nameof(GetRequestBuilder)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/AdaptiveThresholdConfig.cs ================================================ namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public class AdaptiveThresholdConfig { /// /// Represents the base threshold for proxies with a low aggregated result count. /// public int BaseThresholdLowCount { get; set; } = 5; /// /// Represents the base threshold for proxies with a medium aggregated result count. /// public int BaseThresholdMediumCount { get; set; } = 3; /// /// Represents the base threshold for proxies with a high aggregated result count. /// public int BaseThresholdHighCount { get; set; } = 1; /// /// Represents the maximal possible threshold for proxies. /// public int MaxThresholdCount { get; set; } = 15; /// /// Represents the initial learning rate used for updating proxy metrics. /// public double InitialLearningRate { get; set; } = 0.2; /// /// Represents the decay factor applied to the learning rate over time. /// public double LearningRateDecay { get; set; } = 0.01; /// /// Represents the minimum number of calls required before metrics are considered reliable. /// public int MinCallsForReliableMetrics { get; set; } = 5; /// /// Represents the exponential decay rate (per hour) applied to older metrics. /// public double DecayRatePerHour { get; set; } = 0.01; /// /// Stores metrics for each proxy, indexed by the proxy's name. /// public Dictionary ProxyMetrics { get; set; } = []; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/AdaptiveThresholdManager.cs ================================================ using System.Text.Json; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public interface IProvideAdaptiveThreshold { public void LoadConfig(string? configPath); public int GetDynamicThreshold(string proxyName, int aggregatedCount); public void UpdateMetrics(string proxyName, double responseTimeMs, int newCount, bool success); } public class AdaptiveThresholdManager : IProvideAdaptiveThreshold, IDisposable { private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true, WriteIndented = true }; private string? _configFilePath; private bool _disposed; private System.Timers.Timer? _saveTimer; private bool _isDirty; private readonly object _lock = new(); public AdaptiveThresholdConfig Config { get; private set; } = new(); public void LoadConfig(string? configPath) { if (string.IsNullOrWhiteSpace(configPath) || configPath == _configFilePath) return; _configFilePath = configPath; if (!File.Exists(_configFilePath)) return; try { Config = JsonSerializer.Deserialize(File.ReadAllText(_configFilePath)) ?? new AdaptiveThresholdConfig(); _saveTimer = new System.Timers.Timer(300000) { AutoReset = true }; _saveTimer.Elapsed += (s, e) => { if (_isDirty) SaveConfig(); }; _saveTimer.Start(); AppDomain.CurrentDomain.ProcessExit += (s, e) => SaveConfig(); } catch { } } private void SaveConfig() { lock (_lock) { if (!_isDirty || _configFilePath == null) return; string json = JsonSerializer.Serialize(Config, _jsonOptions); File.WriteAllText(_configFilePath, json); _isDirty = false; } } /// /// Applies exponential decay to the stored metrics based on the time elapsed since the last update. /// /// The proxy metrics to decay. private void DecayMetrics(ProxyMetrics metrics) { double hoursElapsed = (DateTime.UtcNow - metrics.LastUpdate).TotalHours; if (hoursElapsed > 0) { double decayFactor = Math.Exp(-Config.DecayRatePerHour * hoursElapsed); metrics.Calls *= decayFactor; metrics.Failures *= decayFactor; metrics.TotalResponseTime *= decayFactor; metrics.TotalNewResults *= decayFactor; metrics.LastUpdate = DateTime.UtcNow; } } /// /// Computes a dynamic threshold for a given proxy based on its historical metrics and the aggregated result count. /// /// The name of the proxy. /// The aggregated result count. /// The computed dynamic threshold. public int GetDynamicThreshold(string proxyName, int aggregatedCount) { int baseThreshold = aggregatedCount < 10 ? Config.BaseThresholdLowCount : aggregatedCount < 50 ? Config.BaseThresholdMediumCount : Config.BaseThresholdHighCount; if (!Config.ProxyMetrics.TryGetValue(proxyName, out ProxyMetrics? metrics) || metrics.Calls < Config.MinCallsForReliableMetrics) return baseThreshold; DecayMetrics(metrics); double failureRatio = metrics.Calls > 0 ? metrics.Failures / metrics.Calls : 0.0; double qualityAdjustment = 1.0 - metrics.QualityScore; double performanceAdjustment = 1.0 - metrics.PerformanceScore; // Weights can be tuned; here we use a weighted sum. double adjustment = (0.5 * qualityAdjustment) + (0.3 * performanceAdjustment) + (0.2 * failureRatio); int maxAdjustment = Config.MaxThresholdCount - baseThreshold; int dynamicThreshold = baseThreshold + (int)Math.Round(adjustment * maxAdjustment); return dynamicThreshold < 1 ? 1 : dynamicThreshold; } /// /// Updates the metrics for a given proxy based on the latest call. /// /// The name of the proxy. /// The response time (in milliseconds) for the call. /// The number of new results returned by the call. /// Whether the call was considered a valid success. public void UpdateMetrics(string proxyName, double responseTimeMs, int newCount, bool success) { lock (_lock) { if (!Config.ProxyMetrics.TryGetValue(proxyName, out ProxyMetrics? metrics)) { metrics = new ProxyMetrics(); Config.ProxyMetrics[proxyName] = metrics; } DecayMetrics(metrics); metrics.Calls++; metrics.TotalResponseTime += responseTimeMs; metrics.TotalNewResults += newCount; if (!success) metrics.Failures++; double learningRate = Config.InitialLearningRate / (1 + (Config.LearningRateDecay * metrics.Calls)); double expectedCount = metrics.ExpectedNewResults; double measuredQuality = expectedCount > 0 ? Math.Min(1.0, newCount / expectedCount) : 0.0; metrics.QualityScore += learningRate * (measuredQuality - metrics.QualityScore); double measuredPerformance = 1.0; if (responseTimeMs > 0) { double resultsPerSecond = newCount / (responseTimeMs / 1000.0); double expectedRate = metrics.AverageResponseTime > 0 ? metrics.ExpectedNewResults / (metrics.AverageResponseTime / 1000.0) : 1.0; if (expectedRate > 0) measuredPerformance = Math.Min(1.0, resultsPerSecond / expectedRate); } metrics.PerformanceScore += learningRate * (measuredPerformance - metrics.PerformanceScore); _isDirty = true; } } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { if (!_disposed) { if (disposing) { SaveConfig(); if (_saveTimer != null) { _saveTimer.Stop(); _saveTimer.Elapsed -= (s, e) => { if (_isDirty) SaveConfig(); }; _saveTimer.Dispose(); _saveTimer = null; } } _disposed = true; } } ~AdaptiveThresholdManager() => Dispose(false); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/ISupportMetadataMixing.cs ================================================ using NzbDrone.Core.Music; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public enum MetadataSupportLevel { Unsupported, ImplicitSupported, Supported } public interface ISupportMetadataMixing : IProxy { MetadataSupportLevel CanHandleSearch(string? albumTitle = null, string? artistName = null); MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds); MetadataSupportLevel CanHandleChanged(); MetadataSupportLevel CanHandleId(string albumId); string? SupportsLink(List links); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/MixedMetadataProxy.cs ================================================ using FuzzySharp; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Datastore; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using System.Reflection; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo))] [ProxyFor(typeof(IProvideAlbumInfo))] [ProxyFor(typeof(ISearchForNewArtist))] [ProxyFor(typeof(ISearchForNewAlbum))] [ProxyFor(typeof(ISearchForNewEntity))] [ProxyFor(typeof(IMetadataRequestBuilder))] public class MixedMetadataProxy : MixedProxyBase { public override string Name => "MetaMix"; private const int ALBUM_SIMILARITY_THRESHOLD = 60; private const int ARTIST_SIMILARITY_THRESHOLD = 70; private const int DEFAULT_PRIORITY = 50; private const int MIN_QUERY_LENGTH = 5; private readonly IArtistService _artistService; internal readonly IProvideAdaptiveThreshold _adaptiveThreshold; public MixedMetadataProxy(Lazy proxyService, IProvideAdaptiveThreshold adaptiveThreshold, IArtistService artistService, Logger logger) : base(proxyService, logger) { _adaptiveThreshold = adaptiveThreshold; _artistService = artistService; InitializeAdaptiveThreshold(); } #region Proxy Methods public Tuple> GetAlbumInfo(string id) { List candidates = GetCandidateProxies(x => x.CanHandleId(id), typeof(IProvideAlbumInfo)); if (candidates.Count == 0) throw new NotImplementedException($"No proxy available to handle album id: {id}"); ProxyCandidate selected = candidates[0]; _logger.Trace($"GetAlbumInfo: Using proxy {selected.Proxy.Name} with priority {selected.Priority} for album id {id}"); return InvokeProxyMethod>>(selected.Proxy, nameof(GetAlbumInfo), id); } public HashSet GetChangedAlbums(DateTime startTime) => GetChanged(x => InvokeProxyMethod>(x, nameof(GetChangedAlbums), startTime)); public Artist GetArtistInfo(string lidarrId, int metadataProfileId) { _logger.Trace($"Fetching artist info for Lidarr ID: {lidarrId} with Metadata Profile ID: {metadataProfileId}"); Artist baseArtist = _artistService.FindById(lidarrId); HashSet usedProxies = []; List newArtists = ExecuteArtistSearch(lidarrId, metadataProfileId, baseArtist, usedProxies); Dictionary> proxyAlbumMap = BuildProxyAlbumMap(baseArtist); return MergeAllArtistData(newArtists, baseArtist, lidarrId, proxyAlbumMap, usedProxies); } public HashSet GetChangedArtists(DateTime startTime) => GetChanged(x => InvokeProxyMethod>(x, nameof(GetChangedArtists), startTime)); public List SearchForNewArtist(string artistName) => new ProxyDecisionHandler( mixedProxy: this, searchExecutor: proxy => InvokeProxyMethod>(proxy, nameof(SearchForNewArtist), artistName), containsItem: ContainsArtist, isValidQuery: () => IsValidQuery(artistName), supportSelector: s => s.CanHandleSearch(artistName: artistName), interfaceType: typeof(ISearchForNewArtist) ).ExecuteSearch(); public List SearchForNewAlbum(string albumTitle, string artistName) => new ProxyDecisionHandler( mixedProxy: this, searchExecutor: proxy => InvokeProxyMethod>(proxy, nameof(SearchForNewAlbum), albumTitle, artistName), containsItem: ContainsAlbum, isValidQuery: () => IsValidQuery(albumTitle, artistName), supportSelector: s => s.CanHandleSearch(albumTitle, artistName), interfaceType: typeof(ISearchForNewAlbum) ).ExecuteSearch(); public List SearchForNewAlbumByRecordingIds(List recordingIds) => new ProxyDecisionHandler( mixedProxy: this, searchExecutor: proxy => InvokeProxyMethod>(proxy, nameof(SearchForNewAlbumByRecordingIds), recordingIds), containsItem: ContainsAlbum, isValidQuery: () => true, supportSelector: s => s.CanHandleIRecordingIds([.. recordingIds]), interfaceType: typeof(ISearchForNewAlbum) ).ExecuteSearch(); public List SearchForNewEntity(string albumTitle) => new ProxyDecisionHandler( mixedProxy: this, searchExecutor: proxy => InvokeProxyMethod>(proxy, nameof(SearchForNewEntity), albumTitle), containsItem: ContainsEntity, isValidQuery: () => IsValidQuery(albumTitle), supportSelector: s => s.CanHandleSearch(albumTitle: albumTitle), interfaceType: typeof(ISearchForNewEntity) ).ExecuteSearch(); private List ExecuteArtistSearch(string lidarrId, int metadataProfileId, Artist? baseArtist, HashSet usedProxies) => new ProxyDecisionHandler( mixedProxy: this, searchExecutor: proxy => ExecuteSingleProxyArtistSearch(proxy, lidarrId, metadataProfileId, baseArtist, usedProxies), containsItem: ContainsArtistInfo, isValidQuery: null, supportSelector: s => DetermineSupportLevel(s, lidarrId, baseArtist), interfaceType: typeof(IProvideArtistInfo) ).ExecuteSearch(); public IHttpRequestBuilderFactory GetRequestBuilder() { List candidates = GetCandidateProxies(_ => MetadataSupportLevel.Supported, typeof(IMetadataRequestBuilder)); if (candidates.Count == 0) throw new InvalidOperationException("No proxy available to handle IMetadataRequestBuilder"); return InvokeProxyMethod(candidates[0].Proxy, nameof(GetRequestBuilder)); } #endregion Proxy Methods private void InitializeAdaptiveThreshold() { if (MixedMetadataProxySettings.Instance?.DynamicThresholdMode == true) _adaptiveThreshold.LoadConfig(MixedMetadataProxySettings.Instance?.WeightsPath); } private List ExecuteSingleProxyArtistSearch(IProxy proxy, string lidarrId, int metadataProfileId, Artist? baseArtist, HashSet usedProxies) { ISupportMetadataMixing mixingProxy = (ISupportMetadataMixing)proxy; _logger.Debug($"Checking proxy: {proxy.Name}"); Artist? proxyArtist = TryGetArtistFromProxy(proxy, mixingProxy, lidarrId, metadataProfileId, baseArtist); if (HasValidAlbumData(proxyArtist)) { _logger.Trace($"Proxy {proxy.Name} retrieved {proxyArtist!.Albums!.Value.Count} albums."); usedProxies.Add(proxy); } return [proxyArtist!]; } private Artist? TryGetArtistFromProxy(IProxy proxy, ISupportMetadataMixing mixingProxy, string lidarrId, int metadataProfileId, Artist? baseArtist) { try { if (mixingProxy.CanHandleId(lidarrId) == MetadataSupportLevel.Supported) { _logger.Debug($"Proxy {proxy.Name} can handle ID {lidarrId} directly."); return InvokeProxyMethod(proxy, nameof(GetArtistInfo), lidarrId, metadataProfileId); } if (baseArtist?.Metadata?.Value?.Links != null) { _logger.Trace($"Checking if proxy {proxy.Name} supports links for base artist {baseArtist.Name}"); string? proxyID = mixingProxy.SupportsLink(baseArtist.Metadata.Value.Links); if (proxyID != null) { _logger.Debug($"Proxy {proxy.Name} found matching link-based ID: {proxyID}"); return InvokeProxyMethod(proxy, nameof(GetArtistInfo), proxyID, metadataProfileId); } } } catch (Exception ex) { _logger.Warn(ex, $"Proxy {proxy.Name} failed to get artist info for {lidarrId}."); } return null; } private MetadataSupportLevel DetermineSupportLevel(ISupportMetadataMixing supportMixing, string lidarrId, Artist? baseArtist) { if (supportMixing.CanHandleId(lidarrId) == MetadataSupportLevel.Supported) return MetadataSupportLevel.Supported; if (MixedMetadataProxySettings.Instance?.PopulateWithMultipleProxies == true) { if (supportMixing.SupportsLink(baseArtist?.Metadata?.Value?.Links ?? []) != null || MixedMetadataProxySettings.Instance?.TryFindArtist == true) return MetadataSupportLevel.ImplicitSupported; } _logger.Trace($"Support selector: {supportMixing.Name} does not support {lidarrId}"); return MetadataSupportLevel.Unsupported; } private Dictionary> BuildProxyAlbumMap(Artist? baseArtist) { Dictionary> proxyAlbumMap = []; if (baseArtist?.Albums?.Value?.Any() != true) return proxyAlbumMap; _logger.Trace($"Base artist has {baseArtist.Albums.Value.Count} albums, checking proxy support..."); foreach (Album? album in baseArtist.Albums.Value) { IProxy? candidate = FindProxyForAlbum(album.ForeignAlbumId); if (candidate != null) { _logger.Trace($"Album '{album.Title}' is supported by proxy: {candidate.Name}"); proxyAlbumMap.TryAdd(candidate, []); proxyAlbumMap[candidate].Add(album); } } return proxyAlbumMap; } private IProxy? FindProxyForAlbum(string foreignAlbumId) => ProxyService.Value.ActiveProxies .OfType() .Cast() .FirstOrDefault(p => ((ISupportMetadataMixing)p).CanHandleId(foreignAlbumId) == MetadataSupportLevel.Supported); private Artist MergeAllArtistData(List newArtists, Artist? baseArtist, string lidarrId, Dictionary> proxyAlbumMap, HashSet usedProxies) { List validArtists = newArtists.Where(x => x != null).ToList(); Artist? mergedArtist = validArtists.Find(x => x.ForeignArtistId == lidarrId) ?? baseArtist ?? validArtists.FirstOrDefault(); if (mergedArtist == null) return null!; AddAlbumsFromUnusedProxies(mergedArtist, proxyAlbumMap, usedProxies); MergeAdditionalArtists(mergedArtist, validArtists); _logger.Info($"Final merged artist: {mergedArtist.Name} with {mergedArtist.Albums?.Value?.Count ?? 0} albums."); return mergedArtist; } private void AddAlbumsFromUnusedProxies(Artist mergedArtist, Dictionary> proxyAlbumMap, HashSet usedProxies) { foreach ((IProxy proxy, List albums) in proxyAlbumMap.Where(kvp => !usedProxies.Contains(kvp.Key))) { _logger.Debug($"Adding old albums from proxy {proxy.Name} to merged artist {mergedArtist.Name}"); AddOldAlbums(mergedArtist, albums); } } private static void MergeAdditionalArtists(Artist mergedArtist, List validArtists) => validArtists.Where(a => a != mergedArtist).ToList() .ForEach(artist => MergeArtists(mergedArtist, artist)); internal List GetCandidateProxies(Func supportSelector, Type interfaceType) { List candidates = ProxyService.Value.ActiveProxies .Where(p => p != this && (p is ISupportMetadataMixing || SupportsInterface(p, interfaceType))) .Select(p => CreateProxyCandidate(p, supportSelector)) .Where(c => c.Support != MetadataSupportLevel.Unsupported) .ToList(); if (candidates.Count == 0) return candidates; BoostInternalProxyPriorities(candidates); return [.. candidates.OrderByDescending(c => c.Support).ThenBy(c => c.Priority)]; } private static ProxyCandidate CreateProxyCandidate(IProxy proxy, Func supportSelector) => new() { Proxy = proxy, Priority = GetPriority(proxy.Name ?? string.Empty), Support = (proxy is ISupportMetadataMixing mixingProxy) ? supportSelector(mixingProxy) : MetadataSupportLevel.Supported }; private static void BoostInternalProxyPriorities(List candidates) { int maxPriority = candidates.Max(c => c.Priority); foreach (ProxyCandidate? candidate in candidates.Where(c => c.Proxy.GetProxyMode() == ProxyMode.Internal)) candidate.Priority = maxPriority; } private HashSet GetChanged(Func> func) { HashSet result = []; List candidates = GetCandidateProxies(x => x.CanHandleChanged(), null!); foreach (IProxy? proxy in candidates.Select(c => c.Proxy)) { try { HashSet changed = func(proxy); if (changed != null) result.UnionWith(changed); } catch (Exception ex) { _logger.Error(ex, $"GetChanged: Exception from proxy {proxy.Name ?? "Unknown"}"); } } return result; } internal int CalculateThreshold(string proxyName, int aggregatedCount) => MixedMetadataProxySettings.Instance?.DynamicThresholdMode == true ? _adaptiveThreshold.GetDynamicThreshold(proxyName, aggregatedCount) : GetThreshold(aggregatedCount); #region Utility Methods private static int GetPriority(string proxyName) { if (string.IsNullOrWhiteSpace(proxyName) || MixedMetadataProxySettings.Instance?.Priotities == null) return DEFAULT_PRIORITY; KeyValuePair matchingPriority = MixedMetadataProxySettings.Instance.Priotities .FirstOrDefault(x => string.Equals(x.Key, proxyName, StringComparison.OrdinalIgnoreCase)); return !string.IsNullOrWhiteSpace(matchingPriority.Value) && int.TryParse(matchingPriority.Value, out int priority) ? priority : DEFAULT_PRIORITY; } private static int GetThreshold(int aggregatedCount) => aggregatedCount switch { < 10 => 5, < 50 => 3, _ => 1 }; private static bool SupportsInterface(IProxy proxy, Type interfaceType) => proxy.GetType().GetCustomAttributes() .Any(attr => attr.OriginalInterface == interfaceType); private static bool HasValidAlbumData(Artist? artist) => artist?.Albums?.Value != null && artist.Albums.Value.Count > 0; private static bool IsValidQuery(params string[] queries) => !queries.Any(query => string.IsNullOrWhiteSpace(query) || query.Length < MIN_QUERY_LENGTH || !query.Any(char.IsLetter)); private static bool ContainsAlbum(List albums, Album newAlbum) => albums.Any(album => Fuzz.Ratio(album.Title, newAlbum.Title) > ALBUM_SIMILARITY_THRESHOLD); private static bool ContainsArtist(List artists, Artist newArtist) => artists.Any(artist => Fuzz.Ratio(artist.Name, newArtist.Name) > ARTIST_SIMILARITY_THRESHOLD); private static bool ContainsArtistInfo(List artists, Artist newArtist) { if (artists.Count == 0 || newArtist?.Albums?.Value == null) return false; return newArtist.Albums.Value.All(album => artists.Any(existing => existing.Albums?.Value != null && ContainsAlbum(existing.Albums.Value, album))); } private static bool ContainsEntity(List entities, object newEntity) => newEntity switch { Album newAlbum => ContainsAlbum(entities.OfType().ToList(), newAlbum), Artist newArtist => ContainsArtist(entities.OfType().ToList(), newArtist), _ => entities.Any(e => string.Equals(e.ToString(), newEntity.ToString(), StringComparison.Ordinal)) }; private static Artist MergeArtists(Artist baseArtist, Artist newArtist) { if (newArtist.Albums?.Value == null) return baseArtist; EnsureAlbumsCollection(baseArtist); MergeLinks(baseArtist, newArtist); MergeAlbums(baseArtist, newArtist); return baseArtist; } private static void EnsureAlbumsCollection(Artist artist) => artist.Albums ??= new LazyLoaded>([]); private static void MergeLinks(Artist baseArtist, Artist newArtist) { List? newLinks = newArtist.Metadata?.Value?.Links? .Where(newLink => baseArtist.Metadata?.Value?.Links?.Any(baseLink => string.Equals(baseLink.Url, newLink.Url, StringComparison.OrdinalIgnoreCase) || string.Equals(baseLink.Name, newLink.Name, StringComparison.OrdinalIgnoreCase)) == false) .ToList(); if (newLinks?.Any() == true) baseArtist.Metadata?.Value?.Links?.AddRange(newLinks); } private static void MergeAlbums(Artist baseArtist, Artist newArtist) { IEnumerable uniqueAlbums = newArtist.Albums!.Value .Where(album => !ContainsAlbum(baseArtist.Albums!.Value, album)); baseArtist.Albums = baseArtist.Albums.Value.Union(uniqueAlbums).ToList(); } private static Artist AddOldAlbums(Artist artist, List oldAlbums) { EnsureAlbumsCollection(artist); IEnumerable uniqueOldAlbums = oldAlbums.Where(album => !ContainsAlbum(artist.Albums!.Value, album)); artist.Albums = artist.Albums.Value.Union(uniqueOldAlbums).ToList(); return artist; } #endregion Utility Methods } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/MixedMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Metadata.Proxy.MetadataProvider.SkyHook; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public class MixedMetadataProxySettingsValidator : AbstractValidator { public MixedMetadataProxySettingsValidator() { // Validate string values (must be integers between 0 and 50). RuleForEach(x => x.Priotities) .Must(kvp => int.TryParse(kvp.Value, out int intValue) && intValue >= 0 && intValue <= 50) .WithMessage("The priotity for a Proxy must be a number between 0 and 50."); // Validate ArtistQueryTimeoutSeconds. RuleFor(x => x.ArtistQueryTimeoutSeconds) .GreaterThan(0) .WithMessage("Artist Query Timeout must be greater than 0 seconds."); // Validate MaxThreshold (must not exceed 25). RuleFor(x => x.MaxThreshold) .InclusiveBetween(1, 25) .WithMessage("Max Threshold must be between 1 and 25."); } } public class MixedMetadataProxySettings : IProviderConfig { private static readonly MixedMetadataProxySettingsValidator Validator = new(); private readonly IEnumerable> _priotities; public static MixedMetadataProxySettings? Instance { get; private set; } public MixedMetadataProxySettings() { _priotities = ProxyServiceStarter.ProxyService?.ActiveProxies? .Where(x => x is ISupportMetadataMixing) .Where(x => x.GetProxyMode() != ProxyMode.Internal) .Select(x => new KeyValuePair(x.Name, x is SkyHookMetadataProxy ? "0" : "50")) .ToList() ?? Enumerable.Empty>(); _customConversion = _priotities.ToList(); Instance = this; ArtistQueryTimeoutSeconds = 30; MaxThreshold = 15; } [FieldDefinition(1, Label = "Priority Rules", Type = FieldType.KeyValueList, Section = MetadataSectionType.Metadata, HelpText = "Define priority rules for proxies. Values must be between 0 and 50.")] public IEnumerable> Priotities { get => _customConversion; set { if (value != null) { Dictionary customDict = value.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); _customConversion = [.. _priotities .Select(kvp => new KeyValuePair(kvp.Key, customDict.TryGetValue(kvp.Key, out string? customValue) ? customValue : kvp.Value)) .OrderBy(x => x.Value)]; } } } private IEnumerable> _customConversion; [FieldDefinition(2, Label = "Maximal usable threshold", Section = MetadataSectionType.Metadata, Type = FieldType.Number, HelpText = "The maximum threshold added to a lower priority proxy to still use for populating data.", Placeholder = "15")] public int MaxThreshold { get; set; } [FieldDefinition(3, Label = "Dynamic threshold", Section = MetadataSectionType.Metadata, Type = FieldType.Checkbox, HelpText = "Generate a dynamic threshold based on old results or use a static one.")] public bool DynamicThresholdMode { get; set; } [FieldDefinition(4, Label = "Storing Path", Section = MetadataSectionType.Metadata, Type = FieldType.Path, HelpText = "Path to store the dynamic threshold for usage after restarts.")] public string WeightsPath { get; set; } = string.Empty; [FieldDefinition(5, Label = "Artist Query Timeout", Unit = "Seconds", Section = MetadataSectionType.Metadata, Type = FieldType.Number, HelpText = "Timeout for artist queries when previous albums exist.", Placeholder = "30")] public int ArtistQueryTimeoutSeconds { get; set; } [FieldDefinition(6, Label = "Multi-Source Population", Section = MetadataSectionType.Metadata, Type = FieldType.Checkbox, HelpText = "Enable queries to multiple metadata providers when populating artist information. Uses fallback strategy when previous album data exists.")] public bool PopulateWithMultipleProxies { get; set; } = true; public bool TryFindArtist { get; internal set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/ProxyDecisionHandler.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using System.Diagnostics; namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public class ProxyDecisionHandler { private readonly MixedMetadataProxy _mixedProxy; private readonly Func> _searchExecutor; private readonly Func, TResult, bool> _containsItem; private readonly Func _isValidQuery; private readonly Func _supportSelector; private readonly Type _interfaceType; private readonly Logger _logger; public ProxyDecisionHandler(MixedMetadataProxy mixedProxy, Func> searchExecutor, Func, TResult, bool> containsItem, Func? isValidQuery, Func supportSelector, Type interfaceType) { _mixedProxy = mixedProxy; _searchExecutor = searchExecutor; _containsItem = containsItem; _isValidQuery = isValidQuery ?? (() => true); _supportSelector = supportSelector; _interfaceType = interfaceType; _logger = NzbDroneLogger.GetLogger(this); } public List ExecuteSearch() { List aggregatedItems = []; int bestPriority = int.MaxValue; foreach (ProxyCandidate candidate in _mixedProxy.GetCandidateProxies(_supportSelector, _interfaceType)) { if (bestPriority == int.MaxValue) { bestPriority = candidate.Priority; } else { int threshold = _mixedProxy.CalculateThreshold(candidate.Proxy.Name, aggregatedItems.Count); if (candidate.Priority > bestPriority + threshold) { _logger.Debug($"Stopping aggregation due to threshold. Candidate proxy {candidate.Proxy.Name} with priority {candidate.Priority} exceeds threshold (threshold={threshold})."); break; } } Stopwatch sw = Stopwatch.StartNew(); List items = _searchExecutor(candidate.Proxy); sw.Stop(); List newItems = items.Where(item => !_containsItem(aggregatedItems, item)).ToList(); aggregatedItems.AddRange(newItems); int newCount = newItems.Count; _logger.Trace($"{candidate.Proxy.Name} returned {items.Count} items, {newCount} new."); bool queryValid = _isValidQuery(); bool success = !queryValid || newCount > 0; _mixedProxy._adaptiveThreshold.UpdateMetrics(candidate.Proxy.Name, sw.Elapsed.TotalMilliseconds, newCount, success); if (newCount == 0 && aggregatedItems.Count != 0) { _logger.Trace($"No new items from proxy {candidate.Proxy.Name}, stopping further calls."); break; } } return aggregatedItems; } } internal record ProxyCandidate { public IProxy Proxy { get; set; } = null!; public int Priority { get; set; } public MetadataSupportLevel Support { get; set; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/Mixed/ProxyMetrics.cs ================================================ namespace Tubifarry.Metadata.Proxy.MetadataProvider.Mixed { public class ProxyMetrics { /// /// Represents the quality score of the proxy on a scale from 0 to 1, where 1.0 is the best. /// public double QualityScore { get; set; } = 1.0; /// /// Represents the performance score of the proxy on a scale from 0 to 1, where 1.0 is the best. /// public double PerformanceScore { get; set; } = 1.0; /// /// Represents the total number of calls made to the proxy. /// public double Calls { get; set; } /// /// Represents the total number of failed calls made to the proxy. /// public double Failures { get; set; } /// /// Represents the total response time (in milliseconds) for all calls made to the proxy. /// public double TotalResponseTime { get; set; } /// /// Represents the total number of new results returned by the proxy. /// public double TotalNewResults { get; set; } /// /// Represents the timestamp of the last update to the metrics, used for decaying old metrics. /// public DateTime LastUpdate { get; set; } = DateTime.UtcNow; /// /// Calculates the average response time (in milliseconds) for calls made to the proxy. /// public double AverageResponseTime => Calls > 0 ? TotalResponseTime / Calls : 0.0; /// /// Calculates the expected number of new results per call based on historical data. /// public double ExpectedNewResults => Calls > 0 ? TotalNewResults / Calls : 3.0; } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/SkyHook/SkyHookMetdadataProxy.cs ================================================ using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Cloud; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.MetadataSource.SkyHook; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Metadata; using System.Text.RegularExpressions; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.MetadataProvider.SkyHook { [Proxy(ProxyMode.Public)] [ProxyFor(typeof(IProvideArtistInfo), 100)] [ProxyFor(typeof(IProvideAlbumInfo), 100)] [ProxyFor(typeof(ISearchForNewArtist), 100)] [ProxyFor(typeof(ISearchForNewAlbum), 100)] [ProxyFor(typeof(ISearchForNewEntity), 100)] [ProxyFor(typeof(IMetadataRequestBuilder), 100)] public partial class SkyHookMetadataProxy : ProxyBase, ISupportMetadataMixing { private readonly SkyHookProxy _skyHookProxy; private readonly IConfigService _configService; private readonly ILidarrCloudRequestBuilder _defaultRequestFactory; public override string Name => "Lidarr Default"; public SkyHookMetadataProxy( IHttpClient httpClient, IMetadataRequestBuilder requestBuilder, IArtistService artistService, IAlbumService albumService, IConfigService configService, ILidarrCloudRequestBuilder defaultRequestBuilder, IMetadataProfileService metadataProfileService, ICacheManager cacheManager, Logger logger) { _skyHookProxy = new SkyHookProxy(httpClient, requestBuilder, artistService, albumService, logger, metadataProfileService, cacheManager); _configService = configService; _defaultRequestFactory = defaultRequestBuilder; } public Artist GetArtistInfo(string lidarrId, int metadataProfileId) => _skyHookProxy.GetArtistInfo(lidarrId, metadataProfileId); public HashSet GetChangedArtists(DateTime startTime) => _skyHookProxy.GetChangedArtists(startTime); public Tuple> GetAlbumInfo(string id) => _skyHookProxy.GetAlbumInfo(id); public HashSet GetChangedAlbums(DateTime startTime) => _skyHookProxy.GetChangedAlbums(startTime); public List SearchForNewArtist(string title) => _skyHookProxy.SearchForNewArtist(title); public List SearchForNewAlbum(string title, string artist) => _skyHookProxy.SearchForNewAlbum(title, artist); public List SearchForNewAlbumByRecordingIds(List recordingIds) => _skyHookProxy.SearchForNewAlbumByRecordingIds(recordingIds); public List SearchForNewEntity(string title) => _skyHookProxy.SearchForNewEntity(ExtractLidarrId(title)); private static string ExtractLidarrId(string title) { if (string.IsNullOrWhiteSpace(title)) return title; Match m = MusicBrainzRegex().Match(title); return m.Success ? $"lidarr:{m.Groups[1].Value}" : title; } public IHttpRequestBuilderFactory GetRequestBuilder() { return _configService.MetadataSource.IsNotNullOrWhiteSpace() ? new HttpRequestBuilder(_configService.MetadataSource.TrimEnd("/") + "/{route}").KeepAlive().CreateFactory() : _defaultRequestFactory.Search; } // ISupportMetadataMixing implementation /// /// Checks if the given id is in MusicBrainz GUID format (and does not contain an '@'). /// Returns Supported if valid; otherwise, Unsupported. /// public MetadataSupportLevel CanHandleSearch(string? albumTitle = null, string? artistName = null) { if (MusicBrainzRegex().Match(albumTitle ?? string.Empty).Success || MusicBrainzRegex().Match(artistName ?? string.Empty).Success) return MetadataSupportLevel.Supported; if (albumTitle != null && _formatRegex.IsMatch(albumTitle) || (artistName != null && _formatRegex.IsMatch(artistName))) return MetadataSupportLevel.Unsupported; return MetadataSupportLevel.Supported; } public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) => MetadataSupportLevel.Supported; public MetadataSupportLevel CanHandleChanged() => MetadataSupportLevel.Supported; public MetadataSupportLevel CanHandleId(string id) { if (string.IsNullOrWhiteSpace(id) || id.Contains('@')) return MetadataSupportLevel.Unsupported; if (_guidRegex.IsMatch(id)) return MetadataSupportLevel.Supported; return MetadataSupportLevel.Unsupported; } /// /// Examines the provided list of links and returns the MusicBrainz GUID if one is found. /// Recognizes URLs such as: /// https://musicbrainz.org/artist/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// https://musicbrainz.org/release/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// https://musicbrainz.org/recording/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /// public string? SupportsLink(List links) { if (links == null || links.Count == 0) return null; foreach (Links link in links) { if (string.IsNullOrWhiteSpace(link.Url)) continue; Match match = MusicBrainzRegex().Match(link.Url); if (match.Success && match.Groups.Count > 1) return match.Groups[1].Value; } return null; } private static readonly Regex _formatRegex = new(@"^\s*\w+:\s*\w+", RegexOptions.Compiled); [GeneratedRegex(@"\b(?:lidarr:|lidarrid:|mbid:|musicbrainz\.org/(?:artist|release|recording|release-group)/)([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b", RegexOptions.IgnoreCase | RegexOptions.Compiled)] private static partial Regex MusicBrainzRegex(); private static readonly Regex _guidRegex = new(@"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", RegexOptions.IgnoreCase | RegexOptions.Compiled); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MetadataProvider/SkyHook/SykHookMetadataProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Metadata.Proxy.MetadataProvider.SkyHook { public class SykHookMetadataProxySettingsValidator : AbstractValidator { } public class SykHookMetadataProxySettings : IProviderConfig { private static readonly SykHookMetadataProxySettingsValidator Validator = new(); [FieldDefinition(99, Label = "Placeholder", Type = FieldType.Textbox, Section = MetadataSectionType.Metadata, Hidden = HiddenType.Hidden)] public string Placeholder { get; set; } = string.Empty; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/Proxy/MixedProxyBase.cs ================================================ using NLog; using NzbDrone.Core.ThingiProvider; using System.Collections.Concurrent; using System.Reflection; namespace Tubifarry.Metadata.Proxy { public abstract class MixedProxyBase : ProxyBase, IMixedProxy where TSettings : class, IProviderConfig, new() { protected readonly Lazy ProxyService; protected readonly Logger _logger; private readonly InternalProxyWrapper _proxyWrapper; protected MixedProxyBase(Lazy proxyService, Logger logger) { ProxyService = proxyService; _logger = logger; _proxyWrapper = new InternalProxyWrapper(proxyService); } protected T InvokeProxyMethod(IProxy proxy, string methodName, params object[] args) => _proxyWrapper.InvokeProxyMethodDirect(proxy, methodName, args); protected void InvokeProxyMethodVoid(IProxy proxy, string methodName, params object[] args) => _proxyWrapper.InvokeProxyMethodVoidDirect(proxy, methodName, args); protected T InvokeProxyMethod(Type interfaceType, string methodName, params object[] args) => _proxyWrapper.InvokeProxyMethod(interfaceType, methodName, args); protected void InvokeProxyMethodVoid(Type interfaceType, string methodName, params object[] args) => _proxyWrapper.InvokeProxyMethodVoid(interfaceType, methodName, args); private class InternalProxyWrapper(Lazy proxyService) : ProxyWrapperBase(proxyService) { private readonly ConcurrentDictionary _directMethodCache = new(); public T InvokeProxyMethodDirect(IProxy proxy, string methodName, params object[] args) { MethodInfo method = GetOrCreateCachedMethod(_directMethodCache, proxy.GetType(), methodName, args); return InvokeWithUnwrapping(method, proxy, args); } public void InvokeProxyMethodVoidDirect(IProxy proxy, string methodName, params object[] args) { MethodInfo method = GetOrCreateCachedMethod(_directMethodCache, proxy.GetType(), methodName, args); InvokeWithUnwrapping(method, proxy, args); } public new T InvokeProxyMethod(Type interfaceType, string methodName, params object[] args) => base.InvokeProxyMethod(interfaceType, methodName, args); public new void InvokeProxyMethodVoid(Type interfaceType, string methodName, params object[] args) => base.InvokeProxyMethodVoid(interfaceType, methodName, args); } } } ================================================ FILE: Tubifarry/Metadata/Proxy/ProxyAttribute.cs ================================================ using System.Reflection; namespace Tubifarry.Metadata.Proxy { [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] public class ProxyAttribute(ProxyMode mode) : Attribute { public ProxyMode Mode { get; } = mode; } [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ProxyForAttribute(Type originalInterface, int priority = 0) : Attribute { public Type OriginalInterface { get; } = originalInterface; public int Priority { get; } = priority; } public static class ProxyAttributeExtensions { public static void ValidateProxyImplementation(this Type proxyType) { IEnumerable<(Type OriginalInterface, int Priority)> declaredInterfaces = proxyType.GetDeclaredInterfaces(); if (!declaredInterfaces.Any()) return; MethodInfo[] publicMethods = proxyType.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly); List missingMethods = declaredInterfaces .SelectMany(interfaceInfo => GetMissingMethodsForInterface(interfaceInfo.OriginalInterface, publicMethods)) .ToList(); if (missingMethods.Count != 0) throw new InvalidOperationException($"Proxy class {proxyType.Name} is missing required methods: {string.Join(", ", missingMethods)}"); } public static IEnumerable<(Type OriginalInterface, int Priority)> GetDeclaredInterfaces(this Type proxyType) => proxyType.GetCustomAttributes() .Select(attr => (attr.OriginalInterface, attr.Priority)); public static ProxyMode GetProxyMode(this IProxy proxy) => proxy.GetType().GetCustomAttribute()?.Mode ?? ProxyMode.Public; public static bool IsProxyForInterface(this Type proxyType, Type originalInterface) => proxyType.GetDeclaredInterfaces().Any(interfaceInfo => interfaceInfo.OriginalInterface == originalInterface); public static int GetPriorityForInterface(this Type proxyType, Type originalInterface) => proxyType.GetDeclaredInterfaces() .FirstOrDefault(interfaceInfo => interfaceInfo.OriginalInterface == originalInterface) .Priority; public static bool HasMatchingMethod(this Type proxyType, MethodInfo interfaceMethod) { MethodInfo[] publicInstanceMethods = proxyType.GetMethods(BindingFlags.Public | BindingFlags.Instance); return publicInstanceMethods.Any(method => IsMethodMatch(method, interfaceMethod)); } private static IEnumerable GetMissingMethodsForInterface(Type originalInterface, MethodInfo[] cachedProxyMethods) => originalInterface.GetMethods() .Where(interfaceMethod => !cachedProxyMethods.Any(proxyMethod => IsMethodMatch(proxyMethod, interfaceMethod))) .Select(interfaceMethod => $"{originalInterface.Name}.{GetMethodSignature(interfaceMethod)}"); private static bool IsMethodMatch(MethodInfo proxyMethod, MethodInfo interfaceMethod) => proxyMethod.Name == interfaceMethod.Name && (proxyMethod.IsGenericMethodDefinition || interfaceMethod.IsGenericMethodDefinition ? IsGenericMethodMatch(proxyMethod, interfaceMethod) : proxyMethod.ReturnType == interfaceMethod.ReturnType && AreParametersMatching(proxyMethod, interfaceMethod)); private static bool IsGenericMethodMatch(MethodInfo proxyMethod, MethodInfo interfaceMethod) { if (proxyMethod.IsGenericMethodDefinition != interfaceMethod.IsGenericMethodDefinition) return false; if (proxyMethod.IsGenericMethodDefinition && interfaceMethod.IsGenericMethodDefinition) { Type[] proxyGenericArgs = proxyMethod.GetGenericArguments(); Type[] interfaceGenericArgs = interfaceMethod.GetGenericArguments(); return proxyGenericArgs.Length == interfaceGenericArgs.Length && AreGenericParametersMatching(proxyMethod, interfaceMethod) && AreGenericTypesCompatible(proxyMethod.ReturnType, interfaceMethod.ReturnType); } return proxyMethod.ReturnType == interfaceMethod.ReturnType && AreParametersMatching(proxyMethod, interfaceMethod); } private static bool AreGenericParametersMatching(MethodInfo proxyMethod, MethodInfo interfaceMethod) { ParameterInfo[] proxyParams = proxyMethod.GetParameters(); ParameterInfo[] interfaceParams = interfaceMethod.GetParameters(); return proxyParams.Length == interfaceParams.Length && proxyParams.Zip(interfaceParams, (proxyParam, interfaceParam) => AreGenericTypesCompatible(proxyParam.ParameterType, interfaceParam.ParameterType)) .All(isMatch => isMatch); } private static bool AreGenericTypesCompatible(Type proxyType, Type interfaceType) { // Exact type match if (proxyType == interfaceType) return true; // Both are generic parameters, check if they're in the same position if (proxyType.IsGenericParameter && interfaceType.IsGenericParameter) return proxyType.GenericParameterPosition == interfaceType.GenericParameterPosition; // Both are generic types, check definition and arguments recursively if (proxyType.IsGenericType && interfaceType.IsGenericType) { if (proxyType.GetGenericTypeDefinition() != interfaceType.GetGenericTypeDefinition()) return false; Type[] proxyArgs = proxyType.GetGenericArguments(); Type[] interfaceArgs = interfaceType.GetGenericArguments(); return proxyArgs.Length == interfaceArgs.Length && proxyArgs.Zip(interfaceArgs, (proxyArg, interfaceArg) => AreGenericTypesCompatible(proxyArg, interfaceArg)) .All(isMatch => isMatch); } // Fallback: check assignability for non-generic parameter types if (proxyType.IsGenericParameter == interfaceType.IsGenericParameter) return proxyType.IsAssignableFrom(interfaceType) || interfaceType.IsAssignableFrom(proxyType); return false; } private static bool AreParametersMatching(MethodInfo proxyMethod, MethodInfo interfaceMethod) { ParameterInfo[] proxyParams = proxyMethod.GetParameters(); ParameterInfo[] interfaceParams = interfaceMethod.GetParameters(); return proxyParams.Length == interfaceParams.Length && proxyParams.Zip(interfaceParams, (proxyParam, interfaceParam) => proxyParam.ParameterType == interfaceParam.ParameterType) .All(isMatch => isMatch); } private static string GetMethodSignature(MethodInfo method) { IEnumerable parameters = method.GetParameters().Select(param => $"{param.ParameterType.Name} {param.Name}"); string genericParams = method.IsGenericMethodDefinition ? $"<{string.Join(", ", method.GetGenericArguments().Select(type => type.Name))}>" : ""; return $"{method.ReturnType.Name} {method.Name}{genericParams}({string.Join(", ", parameters)})"; } } } ================================================ FILE: Tubifarry/Metadata/Proxy/ProxyBase.cs ================================================ using FluentValidation.Results; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Extras.Metadata.Files; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Music; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Metadata.Proxy { public abstract class ProxyBase : IProxy, IMetadata where TSettings : IProviderConfig, new() { public abstract string Name { get; } public Type ConfigContract => typeof(TSettings); public virtual ProviderMessage? Message => null; public IEnumerable DefaultDefinitions => []; public ProviderDefinition? Definition { get; set; } protected TSettings? Settings => Definition?.Settings == null ? default : (TSettings)Definition!.Settings; public virtual object RequestAction(string action, IDictionary query) => default!; public override string ToString() => GetType().Name; // IMetadata implementation public virtual string GetFilenameAfterMove(Artist artist, TrackFile trackFile, MetadataFile metadataFile) => Path.ChangeExtension(trackFile.Path, Path.GetExtension(Path.Combine(artist.Path, metadataFile.RelativePath)).TrimStart('.')); public virtual string GetFilenameAfterMove(Artist artist, string albumPath, MetadataFile metadataFile) => Path.Combine(artist.Path, albumPath, Path.GetFileName(metadataFile.RelativePath)); public virtual MetadataFile FindMetadataFile(Artist artist, string path) => default!; public virtual MetadataFileResult ArtistMetadata(Artist artist) => default!; public virtual MetadataFileResult AlbumMetadata(Artist artist, Album album, string albumPath) => default!; public virtual MetadataFileResult TrackMetadata(Artist artist, TrackFile trackFile) => default!; public virtual List ArtistImages(Artist artist) => []; public virtual List AlbumImages(Artist artist, Album album, string albumPath) => []; public virtual List TrackImages(Artist artist, TrackFile trackFile) => []; public virtual ValidationResult Test() => new(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/ProxyService.cs ================================================ using DryIoc; using NLog; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.ThingiProvider.Events; namespace Tubifarry.Metadata.Proxy { public interface IProxyService { IEnumerable Proxies { get; } IEnumerable ActiveProxies { get; } void RegisterProxy(IProxy proxy); void UnregisterProxy(IProxy proxy); void InitializeProxies(); IProxy? GetActiveProxyForInterface(Type originalInterfaceType); void SetActiveProxy(Type originalInterfaceType, IProxy proxy); } public class ProxyService : IProxyService, IHandle> { private readonly IMetadataFactory _metadataFactory; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; private readonly Dictionary> _interfaceToProxyMap = []; private readonly Dictionary _activeProxyForInterface = []; private readonly List _activeProxies = []; private readonly HashSet _defaultProxies = []; public IEnumerable Proxies { get; private set; } = []; public IEnumerable ActiveProxies => _activeProxies; public ProxyService(IMetadataFactory metadataFactory, IEnumerable proxies, IEventAggregator eventAggregator, Logger logger) { _metadataFactory = metadataFactory; _eventAggregator = eventAggregator; _logger = logger; Proxies = proxies; } public void RegisterProxy(IProxy proxy) { IEnumerable<(Type OriginalInterface, int Priority)> declaredInterfaces = proxy.GetType().GetDeclaredInterfaces(); foreach ((Type originalInterface, int priority) in declaredInterfaces) AddProxyToInterfaceMapping(proxy, originalInterface, priority); _interfaceToProxyMap.Keys.ToList().ForEach(UpdateActiveProxyForInterface); } public void UnregisterProxy(IProxy proxy) { foreach (Type? interfaceType in _interfaceToProxyMap.Keys) RemoveProxyFromInterfaceMapping(proxy, interfaceType); _defaultProxies.Remove(proxy); _logger.Trace($"Unregistered proxy {proxy.Name}"); } public IProxy? GetActiveProxyForInterface(Type originalInterfaceType) => _activeProxyForInterface.GetValueOrDefault(originalInterfaceType) ?? TrySetDefaultProxy(originalInterfaceType); public void SetActiveProxy(Type originalInterfaceType, IProxy proxy) { if (!IsProxyRegisteredForInterface(proxy, originalInterfaceType)) throw new InvalidOperationException($"Proxy {proxy.Name} is not registered for interface {originalInterfaceType.Name}"); _activeProxyForInterface[originalInterfaceType] = proxy; _logger.Trace($"Set active proxy for {originalInterfaceType.Name} to {proxy.Name}"); } public void Handle(ProviderUpdatedEvent message) { if (Proxies.OfType().FirstOrDefault(x => x.Definition?.ImplementationName == message.Definition.ImplementationName) is not IProxy updatedProxy) return; ((IProvider)updatedProxy).Definition = message.Definition; if (message.Definition.Enable) EnableProxy(updatedProxy); else DisableProxy(updatedProxy); } public void InitializeProxies() { _logger.Trace("Initializing proxy system"); IEnumerable metadataProxies = _metadataFactory.All().Select(def => _metadataFactory.GetInstance(def)).OfType(); Proxies = Proxies.Where(p => p is not IMetadata).Concat(metadataProxies).Where(ValidateProxy).DistinctBy(x => x.GetType()).ToArray(); ActivateEnabledProxies(); EnsureDefaultProxies(); _logger.Info($"Initialized proxy system: {_activeProxies.Count} active proxies, {_interfaceToProxyMap.Count} interface mappings"); } private void AddProxyToInterfaceMapping(IProxy proxy, Type originalInterface, int priority) { _interfaceToProxyMap.TryAdd(originalInterface, []); if (!_interfaceToProxyMap[originalInterface].Contains(proxy)) { _interfaceToProxyMap[originalInterface].Add(proxy); _logger.Trace($"Mapped {originalInterface.Name} -> {proxy.Name} (priority: {priority})"); } } private void RemoveProxyFromInterfaceMapping(IProxy proxy, Type interfaceType) { if (!_interfaceToProxyMap.TryGetValue(interfaceType, out List? proxies)) return; proxies.Remove(proxy); if (proxies.Count == 0) RemoveInterfaceMapping(interfaceType); else if (_activeProxyForInterface.GetValueOrDefault(interfaceType) == proxy) SetNewActiveProxyForInterface(interfaceType, proxies); } private void RemoveInterfaceMapping(Type interfaceType) { _interfaceToProxyMap.Remove(interfaceType); _activeProxyForInterface.Remove(interfaceType); _logger.Trace($"Removed all mappings for {interfaceType.Name}"); } private void SetNewActiveProxyForInterface(Type interfaceType, List availableProxies) { IProxy? newActiveProxy = GetHighestPriorityProxy(availableProxies, interfaceType); if (newActiveProxy == null) return; _activeProxyForInterface[interfaceType] = newActiveProxy; _logger.Trace($"Changed active proxy for {interfaceType.Name} to {newActiveProxy.Name}"); } private void UpdateActiveProxyForInterface(Type interfaceType) { List availableProxies = GetActiveProxiesForInterface(interfaceType); IProxy? selectedProxy = SelectOptimalProxy(interfaceType, availableProxies); if (selectedProxy != null) _activeProxyForInterface[interfaceType] = selectedProxy; } private List GetActiveProxiesForInterface(Type interfaceType) => _interfaceToProxyMap.GetValueOrDefault(interfaceType, []) .Where(_activeProxies.Contains) .ToList(); private IProxy? SelectOptimalProxy(Type interfaceType, List availableProxies) { List nonMixedProxies = availableProxies.Where(p => p is not IMixedProxy).ToList(); List mixedProxies = availableProxies.OfType().Cast().ToList(); return nonMixedProxies.Count switch { 0 => SelectDefaultProxy(interfaceType), 1 => SelectSingletonProxy(interfaceType, nonMixedProxies[0]), _ => SelectOrchestratorProxy(interfaceType, nonMixedProxies, mixedProxies) }; } private IProxy? SelectDefaultProxy(Type interfaceType) { IProxy? defaultProxy = GetAllNonMixedProxiesForInterface(interfaceType) .OrderByDescending(p => p.GetType().GetPriorityForInterface(interfaceType)) .FirstOrDefault(); if (defaultProxy != null) _logger.Debug($"No active proxies for {interfaceType.Name}, using default: {defaultProxy.Name}"); return defaultProxy; } private IProxy SelectSingletonProxy(Type interfaceType, IProxy proxy) { _logger.Debug($"Single proxy mode for {interfaceType.Name}: {proxy.Name}"); return proxy; } private IProxy? SelectOrchestratorProxy(Type interfaceType, List nonMixedProxies, List mixedProxies) { IProxy? orchestrator = GetHighestPriorityProxy(mixedProxies, interfaceType); if (orchestrator == null) { List availableOrchestrators = Proxies .Where(p => p is IMixedProxy && p.GetType().IsProxyForInterface(interfaceType)) .ToList(); orchestrator = GetHighestPriorityProxy(availableOrchestrators, interfaceType); } if (orchestrator != null) { _logger.Trace($"Multiple proxies for {interfaceType.Name}, using orchestrator: {orchestrator.Name}"); return orchestrator; } IProxy? fallbackProxy = GetHighestPriorityProxy(nonMixedProxies, interfaceType); _logger.Warn($"Multiple proxies for {interfaceType.Name} but no orchestrator available, using: {fallbackProxy?.Name ?? "(none)"}"); return fallbackProxy; } private IEnumerable GetAllNonMixedProxiesForInterface(Type interfaceType) => _interfaceToProxyMap.GetValueOrDefault(interfaceType, []) .Where(p => p is not IMixedProxy); private static IProxy? GetHighestPriorityProxy(IEnumerable proxies, Type interfaceType) => proxies.OrderByDescending(p => p.GetType().GetPriorityForInterface(interfaceType)).FirstOrDefault(); private IProxy? TrySetDefaultProxy(Type originalInterfaceType) { IProxy? defaultProxy = FindHighestPriorityProxyForInterface(originalInterfaceType); if (defaultProxy == null) { _logger.Warn($"No proxy available for interface {originalInterfaceType.Name}"); return null; } _logger.Trace($"Auto-setting default proxy for {originalInterfaceType.Name}: {defaultProxy.Name}"); EnsureProxyIsActive(defaultProxy); _defaultProxies.Add(defaultProxy); RegisterProxy(defaultProxy); return _activeProxyForInterface.GetValueOrDefault(originalInterfaceType); } private bool IsProxyRegisteredForInterface(IProxy proxy, Type originalInterfaceType) => _interfaceToProxyMap.ContainsKey(originalInterfaceType) && _interfaceToProxyMap[originalInterfaceType].Contains(proxy); private void EnableProxy(IProxy proxy) { if (_activeProxies.Contains(proxy)) return; _activeProxies.Add(proxy); RemoveSupersededDefaultProxies(proxy); RegisterProxy(proxy); _eventAggregator.PublishEvent(new ProxyStatusChangedEvent(proxy, ProxyStatusAction.Enabled)); _logger.Info($"Enabled proxy: {proxy.Name}"); } private void RemoveSupersededDefaultProxies(IProxy newProxy) { foreach (Type? interfaceType in newProxy.GetType().GetDeclaredInterfaces().Select(t => t.OriginalInterface)) { IProxy? currentDefault = _defaultProxies.FirstOrDefault(d => d.GetType().IsProxyForInterface(interfaceType)); if (currentDefault != null) { _activeProxies.Remove(currentDefault); _defaultProxies.Remove(currentDefault); _logger.Trace($"Removed superseded default proxy: {currentDefault.Name}"); } } } private void DisableProxy(IProxy proxy) { if (!_activeProxies.Contains(proxy)) return; _activeProxies.Remove(proxy); _defaultProxies.Remove(proxy); UnregisterProxy(proxy); _interfaceToProxyMap.Keys.ToList().ForEach(UpdateActiveProxyForInterface); _eventAggregator.PublishEvent(new ProxyStatusChangedEvent(proxy, ProxyStatusAction.Disabled)); _logger.Info($"Disabled proxy: {proxy.Name}"); } private bool ValidateProxy(IProxy proxy) { try { proxy.GetType().ValidateProxyImplementation(); _logger.Trace($"Validated proxy implementation: {proxy.Name}"); return true; } catch (InvalidOperationException ex) { _logger.Warn($"{ex.Message}. Proxy will be excluded from the system."); return false; } } private void ActivateEnabledProxies() { foreach (IProxy proxy in Proxies.Where(x => (x as IProvider)?.Definition?.Enable == true)) { ProxyMode mode = proxy.GetProxyMode(); int interfaceCount = proxy.GetType().GetDeclaredInterfaces().Count(); _logger.Trace($"Activating proxy: {proxy.Name} (Mode: {mode}, Interfaces: {interfaceCount})"); _activeProxies.Add(proxy); RegisterProxy(proxy); } } private void EnsureDefaultProxies() { foreach (Type? originalInterface in GetAllDeclaredInterfaces().Where(i => !_activeProxyForInterface.ContainsKey(i))) SetDefaultProxyForInterface(originalInterface); } private IEnumerable GetAllDeclaredInterfaces() => Proxies.SelectMany(proxy => proxy.GetType().GetDeclaredInterfaces()) .Select(tuple => tuple.OriginalInterface) .Distinct(); private void SetDefaultProxyForInterface(Type originalInterface) { IProxy? defaultProxy = FindHighestPriorityProxyForInterface(originalInterface); if (defaultProxy != null) { _logger.Trace($"Setting default proxy for {originalInterface.Name}: {defaultProxy.Name}"); EnsureProxyIsActive(defaultProxy); _defaultProxies.Add(defaultProxy); int priority = defaultProxy.GetType().GetPriorityForInterface(originalInterface); AddProxyToInterfaceMapping(defaultProxy, originalInterface, priority); UpdateActiveProxyForInterface(originalInterface); } else { _logger.Warn($"No proxy available to handle interface {originalInterface.Name}"); } } private void EnsureProxyIsActive(IProxy proxy) { if (!_activeProxies.Contains(proxy)) _activeProxies.Add(proxy); } private IProxy? FindHighestPriorityProxyForInterface(Type originalInterface) => Proxies.Where(proxy => proxy.GetType().IsProxyForInterface(originalInterface)) .OrderByDescending(proxy => proxy.GetType().GetPriorityForInterface(originalInterface)) .FirstOrDefault(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/ProxyServiceStarter.cs ================================================ using NzbDrone.Common.Messaging; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; namespace Tubifarry.Metadata.Proxy { public enum ProxyMode { Public, Internal } public interface IProxy { public string Name { get; } } public interface IMixedProxy : IProxy; public enum ProxyStatusAction { Enabled, Disabled } public class ProxyStatusChangedEvent(IProxy proxy, ProxyStatusAction action) : IEvent { public IProxy Proxy { get; } = proxy; public ProxyStatusAction Action { get; } = action; } public class ProxyServiceStarter : IHandle { public static IProxyService? ProxyService { get; private set; } public ProxyServiceStarter(IProxyService proxyService) => ProxyService = proxyService; public void Handle(ApplicationStartedEvent message) => ProxyService?.InitializeProxies(); } } ================================================ FILE: Tubifarry/Metadata/Proxy/ProxyWrapperBase.cs ================================================ using System.Collections.Concurrent; using System.Reflection; using System.Runtime.ExceptionServices; namespace Tubifarry.Metadata.Proxy { public abstract class ProxyWrapperBase(Lazy proxyService) { protected readonly Lazy ProxyService = proxyService; private readonly ConcurrentDictionary _methodCache = new(); protected T InvokeProxyMethod(Type interfaceType, string methodName, params object[] args) { IProxy activeProxy = GetActiveProxyOrThrow(interfaceType); MethodInfo method = GetOrCreateCachedMethod(_methodCache, activeProxy.GetType(), methodName, args); return InvokeWithUnwrapping(method, activeProxy, args); } protected void InvokeProxyMethodVoid(Type interfaceType, string methodName, params object[] args) { IProxy activeProxy = GetActiveProxyOrThrow(interfaceType); MethodInfo method = GetOrCreateCachedMethod(_methodCache, activeProxy.GetType(), methodName, args); InvokeWithUnwrapping(method, activeProxy, args); } protected static T InvokeWithUnwrapping(MethodInfo method, object target, object[] args) { try { return (T)method.Invoke(target, args)!; } catch (TargetInvocationException ex) when (ex.InnerException != null) { ExceptionDispatchInfo.Capture(ex.InnerException).Throw(); throw; } } protected static void InvokeWithUnwrapping(MethodInfo method, object target, object[] args) => InvokeWithUnwrapping(method, target, args); private IProxy GetActiveProxyOrThrow(Type interfaceType) => ProxyService.Value.GetActiveProxyForInterface(interfaceType) ?? throw new InvalidOperationException($"No active proxy found for interface {interfaceType.Name}"); protected static MethodInfo GetOrCreateCachedMethod(ConcurrentDictionary cache, Type proxyType, string methodName, object[] args) { string cacheKey = GenerateCacheKey(proxyType, methodName, args); return cache.GetOrAdd(cacheKey, _ => FindAndPrepareMethod(proxyType, methodName, args)); } private static MethodInfo FindAndPrepareMethod(Type proxyType, string methodName, object[] args) { MethodInfo method = FindMatchingMethod(proxyType, methodName, args); if (method.IsGenericMethodDefinition) { Type[] genericArguments = InferGenericArguments(method, args); method = method.MakeGenericMethod(genericArguments); } return method; } private static MethodInfo FindMatchingMethod(Type proxyType, string methodName, object[] args) { Type[] parameterTypes = args.Select(arg => arg?.GetType() ?? typeof(object)).ToArray(); MethodInfo? method = proxyType.GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance, null, parameterTypes, null); if (method?.IsGenericMethodDefinition == false) return method; IEnumerable candidateMethods = proxyType.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => m.Name == methodName && m.GetParameters().Length == args.Length); method = candidateMethods.Where(m => !m.IsGenericMethodDefinition) .FirstOrDefault(m => IsParameterCompatible(m.GetParameters(), args)) ?? candidateMethods.FirstOrDefault(m => m.IsGenericMethodDefinition); return method ?? throw new InvalidOperationException($"Method {methodName} not found on {proxyType.Name}"); } private static bool IsParameterCompatible(ParameterInfo[] parameters, object[] args) { for (int i = 0; i < parameters.Length; i++) { Type paramType = parameters[i].ParameterType; Type? argType = args[i]?.GetType(); if (argType == null) { if (paramType.IsValueType && !IsNullable(paramType)) return false; } else if (!paramType.IsAssignableFrom(argType) && !IsGenericTypeCompatible(paramType, argType)) { return false; } } return true; } private static bool IsGenericTypeCompatible(Type paramType, Type argType) { if (paramType.IsGenericParameter) return true; if (paramType.IsGenericType) { Type genericDefinition = paramType.GetGenericTypeDefinition(); return HasGenericBase(argType, genericDefinition); } return paramType.IsAssignableFrom(argType); } private static bool HasGenericBase(Type type, Type genericDefinition) { Type? current = type; while (current != null) { if (current.IsGenericType && current.GetGenericTypeDefinition() == genericDefinition) return true; current = current.BaseType; } return type.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == genericDefinition); } private static Type[] InferGenericArguments(MethodInfo genericMethod, object[] args) { ParameterInfo[] parameters = genericMethod.GetParameters(); Type[] genericParameters = genericMethod.GetGenericArguments(); Type[] inferredTypes = new Type[genericParameters.Length]; for (int i = 0; i < parameters.Length && i < args.Length; i++) { if (args[i] != null) { InferFromParameter(parameters[i].ParameterType, args[i].GetType(), genericParameters, inferredTypes); } } for (int i = 0; i < inferredTypes.Length; i++) inferredTypes[i] ??= GetDefaultTypeForGeneric(genericParameters[i]); return inferredTypes; } private static void InferFromParameter(Type paramType, Type argType, Type[] genericParams, Type[] inferredTypes) { if (paramType.IsGenericParameter) { int index = Array.IndexOf(genericParams, paramType); if (index >= 0) inferredTypes[index] ??= argType; return; } if (paramType.IsGenericType) { Type paramGenericDef = paramType.GetGenericTypeDefinition(); Type? matchingArgType = FindMatchingGenericType(argType, paramGenericDef); if (matchingArgType != null) { Type[] paramArgs = paramType.GetGenericArguments(); Type[] argArgs = matchingArgType.GetGenericArguments(); for (int i = 0; i < Math.Min(paramArgs.Length, argArgs.Length); i++) { InferFromParameter(paramArgs[i], argArgs[i], genericParams, inferredTypes); } } } } private static Type? FindMatchingGenericType(Type argType, Type genericDefinition) { if (argType.IsGenericType && argType.GetGenericTypeDefinition() == genericDefinition) return argType; Type? current = argType.BaseType; while (current != null) { if (current.IsGenericType && current.GetGenericTypeDefinition() == genericDefinition) return current; current = current.BaseType; } return argType.GetInterfaces() .Where(i => i.IsGenericType) .FirstOrDefault(i => i.GetGenericTypeDefinition() == genericDefinition); } private static Type GetDefaultTypeForGeneric(Type genericParameter) { Type[] constraints = genericParameter.GetGenericParameterConstraints(); return constraints.FirstOrDefault(c => !c.IsInterface) ?? constraints.FirstOrDefault() ?? typeof(object); } private static bool IsNullable(Type type) => type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); private static string GenerateCacheKey(Type proxyType, string methodName, object[] args) => $"{proxyType.FullName}.{methodName}({string.Join(",", args.Select(arg => arg?.GetType().FullName ?? "null"))})"; } } ================================================ FILE: Tubifarry/Metadata/Proxy/RecommendArtists/LastFmSimilarArtistsService.cs ================================================ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Datastore; using NzbDrone.Core.ImportLists.LastFm; using NzbDrone.Core.MediaCover; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using System.Text.Json; using Tubifarry.Core.Utilities; using Tubifarry.ImportLists.LastFmRecommendation; using Tubifarry.Metadata.Proxy.MetadataProvider.Lastfm; namespace Tubifarry.Metadata.Proxy.RecommendArtists { public interface ILastFmSimilarArtistsService { public List GetSimilarArtistsWithMetadata(string artistIdentifier, SimilarArtistsProxySettings settings); } /// /// Service for fetching similar artists from Last.fm API with full metadata /// public class LastFmSimilarArtistsService : ILastFmSimilarArtistsService { private readonly IHttpClient _httpClient; private readonly Logger _logger; private CacheService _cache = null!; private LastfmImageScraper _imageScraper = null!; private LastfmApiService _apiService = null!; private const string LASTFM_API_BASE = "https://ws.audioscrobbler.com/2.0/"; public LastFmSimilarArtistsService(IHttpClient httpClient, Logger logger) { _httpClient = httpClient; _logger = logger; } public List GetSimilarArtistsWithMetadata(string artistIdentifier, SimilarArtistsProxySettings settings) { if (string.IsNullOrWhiteSpace(artistIdentifier)) { _logger.Warn("Artist identifier is empty, cannot fetch similar artists"); return []; } if (string.IsNullOrWhiteSpace(settings?.ApiKey)) { _logger.Warn("Last.fm API key not configured"); return []; } InitServices(settings); List similarArtists = FetchSimilarArtistsFromApi(artistIdentifier, settings, settings.ResultLimit); if (similarArtists.Count == 0) { _logger.Debug($"No similar artists found for: {artistIdentifier}"); return []; } _logger.Trace($"Found {similarArtists.Count} similar artists for '{artistIdentifier}', fetching detailed info..."); List results = []; foreach (LastFmArtist similarArtist in similarArtists) { try { // Get detailed artist info from Last.fm API with caching LastfmArtist? detailedArtist = _cache.FetchAndCacheAsync( $"artist:{similarArtist.Name}", () => _apiService.GetArtistInfoAsync(similarArtist.Name) ).GetAwaiter().GetResult(); if (detailedArtist == null) continue; if (string.IsNullOrWhiteSpace(detailedArtist.MBID)) continue; Artist artist = MapArtistFromLastfmArtist(detailedArtist, artistIdentifier, settings, _imageScraper); results.Add(artist); } catch (Exception ex) { _logger.Warn(ex, $"Failed to process artist: {similarArtist.Name}"); } } _logger.Trace($"Returning {results.Count} similar artists with valid MusicBrainz IDs for '{artistIdentifier}'"); return results; } private void InitServices(SimilarArtistsProxySettings settings) { _cache ??= new CacheService { CacheDuration = TimeSpan.FromDays(21), CacheDirectory = settings.CacheDirectory, CacheType = (CacheType)settings.RequestCacheType, }; _imageScraper ??= new LastfmImageScraper(_httpClient, Tubifarry.UserAgent, _cache); _apiService ??= new LastfmApiService(_httpClient, Tubifarry.UserAgent) { ApiKey = settings.ApiKey, PageSize = 50, MaxPageLimit = 1 }; } private List FetchSimilarArtistsFromApi(string artistIdentifier, SimilarArtistsProxySettings settings, int limit) { try { bool isMbid = Guid.TryParse(artistIdentifier, out _); _logger.Trace($"Fetching similar artists for: {artistIdentifier} (using {(isMbid ? "MBID" : "artist name")})"); HttpRequestBuilder requestBuilder = new HttpRequestBuilder(LASTFM_API_BASE) .AddQueryParam("method", "artist.getSimilar") .AddQueryParam("api_key", settings.ApiKey) .AddQueryParam("format", "json") .AddQueryParam("limit", limit) .SetHeader("User-Agent", Tubifarry.UserAgent) .Accept(HttpAccept.Json); if (isMbid) requestBuilder.AddQueryParam("mbid", artistIdentifier); else requestBuilder.AddQueryParam("artist", artistIdentifier); HttpRequest request = requestBuilder.Build(); HttpResponse response = _httpClient.Get(request); if (response.HasHttpError) { _logger.Warn($"HTTP error fetching similar artists for: {artistIdentifier}"); return []; } LastFmSimilarArtistsResponse? apiResponse = JsonSerializer.Deserialize(response.Content, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); if (apiResponse?.SimilarArtists?.Artist?.Count > 0) { _logger.Debug($"Found {apiResponse.SimilarArtists.Artist.Count} similar artists for '{artistIdentifier}'"); return apiResponse.SimilarArtists.Artist; } _logger.Debug($"No similar artists found for: {artistIdentifier}"); return []; } catch (Exception ex) { _logger.Error(ex, $"Error fetching similar artists for: {artistIdentifier}"); return []; } } /// /// Maps a Last.fm artist to a Lidarr Artist object using the MusicBrainz ID /// private Artist MapArtistFromLastfmArtist(LastfmArtist lastfmArtist, string sourceArtistIdentifier, SimilarArtistsProxySettings settings, LastfmImageScraper imageScraper) { string foreignArtistId = lastfmArtist.MBID; ArtistMetadata metadata = new() { ForeignArtistId = foreignArtistId, Name = lastfmArtist.Name ?? string.Empty, Links = [ new Links { Url = lastfmArtist.Url, Name = "Last.fm" }, new Links { Url = $"https://musicbrainz.org/artist/{foreignArtistId}", Name = "MusicBrainz" } ], Genres = lastfmArtist.Tags?.Tag?.Select(t => t.Name).ToList() ?? [], Status = ArtistStatusType.Continuing, Type = string.Empty, Aliases = [] }; // Set overview if (lastfmArtist.Bio != null && !string.IsNullOrEmpty(lastfmArtist.Bio.Summary)) { metadata.Overview = $"Artist similar to {sourceArtistIdentifier}. {lastfmArtist.Bio.Summary}"; } else if (lastfmArtist.Stats != null) { List overviewParts = [$"Artist similar to {sourceArtistIdentifier}"]; if (lastfmArtist.Stats.PlayCount > 0) overviewParts.Add($"Playcount: {lastfmArtist.Stats.PlayCount:N0}"); if (!string.IsNullOrEmpty(lastfmArtist.Stats.Listeners)) overviewParts.Add($"Listeners: {int.Parse(lastfmArtist.Stats.Listeners):N0}"); metadata.Overview = string.Join(" • ", overviewParts); } else { metadata.Overview = $"Artist similar to {sourceArtistIdentifier}. Found on Last.fm"; } // Calculate rating metadata.Ratings = LastfmMappingHelper.ComputeLastfmRating(lastfmArtist.Stats?.Listeners ?? "0", lastfmArtist.Stats?.PlayCount ?? 0); // Fetch images if enabled if (settings.FetchImages) metadata.Images = FetchArtistImages(lastfmArtist.Name!, imageScraper); else metadata.Images = MapLastfmImages(lastfmArtist.Images); return new Artist { ForeignArtistId = foreignArtistId, Name = lastfmArtist.Name, SortName = lastfmArtist.Name, CleanName = lastfmArtist.Name.CleanArtistName(), Monitored = false, Metadata = new LazyLoaded(metadata), Albums = new LazyLoaded>([]) }; } /// /// Fetches artist images using web scraping /// private List FetchArtistImages(string artistName, LastfmImageScraper imageScraper) { try { _logger.Trace($"Fetching images for artist: {artistName}"); List imageUrls = imageScraper.GetArtistImagesAsync(artistName).GetAwaiter().GetResult(); if (imageUrls.Count == 0) { _logger.Trace($"No images found for artist: {artistName}"); return []; } List mediaCovers = []; for (int i = 0; i < Math.Min(imageUrls.Count, 4); i++) { MediaCoverTypes coverType = i switch { 0 => MediaCoverTypes.Poster, 1 => MediaCoverTypes.Fanart, 2 => MediaCoverTypes.Banner, _ => MediaCoverTypes.Cover }; mediaCovers.Add(new MediaCover(coverType, imageUrls[i])); } _logger.Trace("Fetched {0} images for {1}", mediaCovers.Count, artistName); return mediaCovers; } catch (Exception ex) { _logger.Error(ex, $"Failed to fetch images for artist: {artistName}"); return []; } } /// /// Maps Last.fm API images to MediaCover objects /// private static List MapLastfmImages(List? images) => images? .Where(i => !string.IsNullOrEmpty(i.Url)) .Select(i => new MediaCover { Url = i.Url, CoverType = MapImageSize(i.Size) }).ToList() ?? []; /// /// Maps Last.fm image size to MediaCoverTypes /// private static MediaCoverTypes MapImageSize(string size) => size?.ToLowerInvariant() switch { "mega" or "extralarge" => MediaCoverTypes.Poster, "large" => MediaCoverTypes.Fanart, "medium" => MediaCoverTypes.Headshot, "small" => MediaCoverTypes.Logo, _ => MediaCoverTypes.Poster }; } } ================================================ FILE: Tubifarry/Metadata/Proxy/RecommendArtists/SimilarArtistsProxy.cs ================================================ using NLog; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Music; using Tubifarry.Metadata.Proxy.MetadataProvider.Mixed; namespace Tubifarry.Metadata.Proxy.RecommendArtists { /// /// Proxy for injecting Last.fm similar artists into search results /// [Proxy(ProxyMode.Internal)] [ProxyFor(typeof(ISearchForNewEntity))] public class SimilarArtistsProxy( ILastFmSimilarArtistsService lastFmService, Logger logger) : ProxyBase, ISupportMetadataMixing { private readonly ILastFmSimilarArtistsService _lastFmService = lastFmService; private readonly Logger _logger = logger; public override string Name => "Similar Artists (Last.fm)"; private static readonly string[] SIMILAR_SEARCH_PREFIX = ["similar:", "~"]; public List SearchForNewEntity(string title) { _logger.Trace($"SearchForNewEntity called with: {title}"); if (string.IsNullOrWhiteSpace(title)) { _logger.Trace("Search title is null or empty"); return []; } string? matchedPrefix = SIMILAR_SEARCH_PREFIX.FirstOrDefault(prefix => title.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); if (matchedPrefix == null) { _logger.Trace("Search doesn't use 'similar:' prefix, skipping"); return []; } string targetArtistIdentifier = title[matchedPrefix.Length..].Trim(); if (string.IsNullOrWhiteSpace(targetArtistIdentifier)) { _logger.Trace("No artist identifier provided after prefix"); return []; } return _lastFmService .GetSimilarArtistsWithMetadata(targetArtistIdentifier, Settings!) .Cast() .ToList(); } public MetadataSupportLevel CanHandleSearch(string? albumTitle = null, string? artistName = null) => SIMILAR_SEARCH_PREFIX.Any(prefix => albumTitle?.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true || artistName?.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true) ? MetadataSupportLevel.Supported : MetadataSupportLevel.Unsupported; public MetadataSupportLevel CanHandleId(string id) => MetadataSupportLevel.Unsupported; public MetadataSupportLevel CanHandleIRecordingIds(params string[] recordingIds) => MetadataSupportLevel.Unsupported; public MetadataSupportLevel CanHandleChanged() => MetadataSupportLevel.Unsupported; public string? SupportsLink(List links) => null; } } ================================================ FILE: Tubifarry/Metadata/Proxy/RecommendArtists/SimilarArtistsProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.ImportLists.LastFm; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.Proxy.RecommendArtists { public class SimilarArtistsProxySettingsValidator : AbstractValidator { public SimilarArtistsProxySettingsValidator() { // Validate that the API key is not empty when enabled RuleFor(c => c.ApiKey) .NotEmpty() .WithMessage("Last.fm API Key is required when Similar Artists feature is enabled"); // Validate result limit RuleFor(c => c.ResultLimit) .InclusiveBetween(1, 50) .WithMessage("Result limit must be between 1 and 50"); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(1)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); // When using Permanent cache, require a valid CacheDirectory RuleFor(x => x.CacheDirectory) .Must((settings, path) => settings.RequestCacheType != (int)CacheType.Permanent || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate the system stability for Memory cache RuleFor(x => x.RequestCacheType) .Must((type) => type == (int)CacheType.Permanent || Tubifarry.AverageRuntime > TimeSpan.FromDays(1)) .When(x => x.RequestCacheType == (int)CacheType.Memory) .WithMessage("The system is not detected as stable. Please wait for the system to stabilize or use permanent cache."); } } public class SimilarArtistsProxySettings : IProviderConfig { private static readonly SimilarArtistsProxySettingsValidator Validator = new(); public SimilarArtistsProxySettings() { ApiKey = new LastFmUserSettings().ApiKey; ResultLimit = 10; FetchImages = true; Instance = this; } [FieldDefinition(0, Label = "Last.fm API Key", Type = FieldType.Textbox, Section = MetadataSectionType.Metadata, HelpText = "Your Last.fm API key for fetching similar artists", Privacy = PrivacyLevel.ApiKey)] public string ApiKey { get; set; } [FieldDefinition(1, Label = "Result Limit", Type = FieldType.Number, Section = MetadataSectionType.Metadata, HelpText = "Maximum number of similar artists to return (1-50). Only artists with MusicBrainz IDs are returned.", Placeholder = "10")] public int ResultLimit { get; set; } [FieldDefinition(2, Label = "Fetch Images", Type = FieldType.Checkbox, Section = MetadataSectionType.Metadata, HelpText = "Scrape artist images from Last.fm web pages")] public bool FetchImages { get; set; } [FieldDefinition(3, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "Select Memory (non-permanent) or Permanent caching")] public int RequestCacheType { get; set; } = (int)CacheType.Permanent; [FieldDefinition(4, Label = "Cache Directory", Type = FieldType.Path, HelpText = "Directory to store cached data (only used for Permanent caching)")] public string CacheDirectory { get; set; } = string.Empty; public static SimilarArtistsProxySettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/IProvideScheduledTask.cs ================================================ using NzbDrone.Core.Messaging.Commands; namespace Tubifarry.Metadata.ScheduledTasks { /// /// Interface for providers that can register scheduled tasks. /// Metadata providers implementing this interface will automatically /// have their tasks registered when enabled. /// public interface IProvideScheduledTask { /// /// The command type that will be executed on schedule. /// Type CommandType { get; } /// /// The interval in minutes between task executions. /// int IntervalMinutes { get; } /// /// The priority of the command execution. /// CommandPriority Priority { get; } } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/ScheduledTaskBase.cs ================================================ using FluentValidation.Results; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Extras.Metadata.Files; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Music; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Metadata.ScheduledTasks { /// /// Base class for metadata providers that want to register scheduled tasks. /// Provides automatic task registration/unregistration when enabled/disabled. /// /// The settings type for this scheduled task provider. public abstract class ScheduledTaskBase : IProvideScheduledTask, IMetadata where TSettings : IProviderConfig, new() { public abstract string Name { get; } public Type ConfigContract => typeof(TSettings); public virtual ProviderMessage? Message => null; public IEnumerable DefaultDefinitions => []; public ProviderDefinition? Definition { get; set; } protected virtual TSettings? Settings => Definition?.Settings == null ? default : (TSettings)Definition.Settings; public abstract Type CommandType { get; } public abstract int IntervalMinutes { get; } public virtual CommandPriority Priority => CommandPriority.Low; public virtual object RequestAction(string action, IDictionary query) => default!; public virtual ValidationResult Test() => new(); public virtual string GetFilenameAfterMove(Artist artist, TrackFile trackFile, MetadataFile metadataFile) => Path.ChangeExtension(trackFile.Path, Path.GetExtension(Path.Combine(artist.Path, metadataFile.RelativePath)).TrimStart('.')); public virtual string GetFilenameAfterMove(Artist artist, string albumPath, MetadataFile metadataFile) => Path.Combine(artist.Path, albumPath, Path.GetFileName(metadataFile.RelativePath)); public virtual MetadataFile FindMetadataFile(Artist artist, string path) => default!; public virtual MetadataFileResult ArtistMetadata(Artist artist) => default!; public virtual MetadataFileResult AlbumMetadata(Artist artist, Album album, string albumPath) => default!; public virtual MetadataFileResult TrackMetadata(Artist artist, TrackFile trackFile) => default!; public virtual List ArtistImages(Artist artist) => []; public virtual List AlbumImages(Artist artist, Album album, string albumPath) => []; public virtual List TrackImages(Artist artist, TrackFile trackFile) => []; public override string ToString() => Name; } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/ScheduledTaskService.cs ================================================ using NLog; using NzbDrone.Common.Cache; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Jobs; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.ThingiProvider.Events; namespace Tubifarry.Metadata.ScheduledTasks { public interface IScheduledTaskService { IEnumerable TaskProviders { get; } IEnumerable ActiveTaskProviders { get; } void InitializeTasks(); void UpdateTask() where T : IProvideScheduledTask; void EnableTask(IProvideScheduledTask provider); void DisableTask(IProvideScheduledTask provider); } public enum TaskStatusAction { Enabled, Disabled, IntervalUpdated } public class ScheduledTaskService( IScheduledTaskRepository _scheduledTaskRepository, IMetadataFactory _metadataFactory, ICacheManager _cacheManager, Logger _logger) : IScheduledTaskService, IHandle>, IHandle>, IHandle> { private readonly Dictionary _registeredTasks = []; private readonly List _activeTaskProviders = []; public IEnumerable TaskProviders { get; private set; } = []; public IEnumerable ActiveTaskProviders => _activeTaskProviders; public void InitializeTasks() { _logger.Trace("Initializing scheduled task system"); IEnumerable taskProviders = _metadataFactory.GetAvailableProviders() .OfType() .Where(ValidateTaskProvider) .DistinctBy(x => x.CommandType.FullName) .ToArray(); TaskProviders = taskProviders; foreach (IProvideScheduledTask provider in taskProviders.Where(x => (x as IProvider)?.Definition?.Enable == true)) EnableTask(provider); _logger.Debug($"Initialized scheduled task system: {_activeTaskProviders.Count} active tasks, {TaskProviders.Count()} total task providers"); } public void Handle(ProviderUpdatedEvent message) { if (TaskProviders.FirstOrDefault(x => (x as IMetadata)?.Definition?.ImplementationName == message.Definition.ImplementationName) is not IProvideScheduledTask taskProvider) { return; } _logger.Trace($"Provider updated event for: {(taskProvider as IProvider)?.Name}, Enabled: {message.Definition.Enable}"); if (message.Definition.Enable) EnableTask(taskProvider); else DisableTask(taskProvider); } public void Handle(ProviderAddedEvent message) { if (message.Definition.Implementation == null) return; IProvideScheduledTask? taskProvider = TaskProviders.FirstOrDefault(x => (x as IProvider)?.Definition?.ImplementationName == message.Definition.ImplementationName); if (taskProvider != null && message.Definition.Enable) EnableTask(taskProvider); } public void Handle(ProviderDeletedEvent message) { IProvideScheduledTask? taskProvider = _activeTaskProviders.FirstOrDefault(x => (x as IProvider)?.Definition?.Id == message.ProviderId); if (taskProvider != null) DisableTask(taskProvider); } public void EnableTask(IProvideScheduledTask provider) { if (provider.IntervalMinutes <= 0) { _logger.Trace($"Task has interval <= 0, treating as disabled: {(provider as IProvider)?.Name}"); DisableTask(provider); return; } if (_activeTaskProviders.Contains(provider)) { _logger.Trace($"Task already enabled: {(provider as IProvider)?.Name}"); UpdateTaskInterval(provider); return; } _activeTaskProviders.Add(provider); RegisterTask(provider); _logger.Info($"Enabled scheduled task: {(provider as IProvider)?.Name} (Interval: {provider.IntervalMinutes}m, Priority: {provider.Priority})"); } public void DisableTask(IProvideScheduledTask provider) { if (!_activeTaskProviders.Contains(provider)) { _logger.Debug($"Task already disabled: {(provider as IProvider)?.Name}"); return; } string typeName = provider.CommandType.FullName!; try { ScheduledTask? existing = _scheduledTaskRepository.All().SingleOrDefault(t => t.TypeName == typeName); if (existing != null) { _scheduledTaskRepository.Delete(existing.Id); RemoveFromCache(typeName); _logger.Debug($"Deleted scheduled task from repository: {(provider as IProvider)?.Name}"); } _registeredTasks.Remove(typeName); _activeTaskProviders.Remove(provider); _logger.Info($"Disabled scheduled task: {(provider as IProvider)?.Name}"); } catch (Exception ex) { _logger.Error(ex, $"Failed to disable scheduled task: {(provider as IProvider)?.Name}"); } } private void RegisterTask(IProvideScheduledTask provider) { string typeName = provider.CommandType.FullName!; try { ScheduledTask task = new() { Interval = provider.IntervalMinutes, TypeName = typeName, Priority = provider.Priority }; ScheduledTask? existing = _scheduledTaskRepository.All().SingleOrDefault(t => t.TypeName == typeName); if (existing != null) { existing.Interval = task.Interval; existing.Priority = task.Priority; _scheduledTaskRepository.Update(existing); _logger.Trace($"Updated existing scheduled task: {(provider as IProvider)?.Name}"); } else { task.LastExecution = DateTime.UtcNow; _scheduledTaskRepository.Insert(task); _logger.Trace($"Inserted new scheduled task: {(provider as IProvider)?.Name}"); } UpdateCache(existing ?? task); _registeredTasks[typeName] = provider; } catch (Exception ex) { _logger.Error(ex, $"Failed to register scheduled task: {(provider as IProvider)?.Name}"); } } public void UpdateTask() where T : IProvideScheduledTask { string typeName = typeof(T).FullName!; if (!_registeredTasks.TryGetValue(typeName, out IProvideScheduledTask? provider)) { _logger.Warn($"Cannot update interval: Task provider {typeName} is not registered"); return; } if (provider is not T) { _logger.Warn($"Cannot update interval: Registered provider for {typeName} is not of expected type"); return; } EnableTask(provider); } private void UpdateTaskInterval(IProvideScheduledTask provider) { string typeName = provider.CommandType.FullName!; try { ScheduledTask? existing = _scheduledTaskRepository.All().SingleOrDefault(t => t.TypeName == typeName); if (existing != null && existing.Interval != provider.IntervalMinutes) { existing.Interval = provider.IntervalMinutes; _scheduledTaskRepository.Update(existing); UpdateCache(existing); _logger.Trace($"Updated task interval for {(provider as IProvider)?.Name}: {provider.IntervalMinutes}m"); } } catch (Exception ex) { _logger.Error(ex, $"Failed to update task interval for: {(provider as IProvider)?.Name}"); } } private void UpdateCache(ScheduledTask task) { ICached cache = _cacheManager.GetCache(typeof(TaskManager)); cache.Set(task.TypeName, task); } private void RemoveFromCache(string typeName) { ICached cache = _cacheManager.GetCache(typeof(TaskManager)); cache.Remove(typeName); } private bool ValidateTaskProvider(IProvideScheduledTask provider) { try { if (provider.CommandType == null) { _logger.Warn($"Task provider {(provider as IProvider)?.Name} has no command type. Provider will be excluded."); return false; } if (provider.IntervalMinutes < 0) { _logger.Warn($"Task provider {(provider as IProvider)?.Name} has invalid interval ({provider.IntervalMinutes}m). Provider will be excluded."); return false; } _logger.Trace($"Validated task provider: {(provider as IProvider)?.Name}"); return true; } catch (Exception ex) { _logger.Warn(ex, $"Failed to validate task provider {(provider as IProvider)?.Name}. Provider will be excluded."); return false; } } } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/ScheduledTaskServiceStarter.cs ================================================ using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; namespace Tubifarry.Metadata.ScheduledTasks { public class ScheduledTaskServiceStarter : IHandle { public static IScheduledTaskService? TaskService { get; private set; } public ScheduledTaskServiceStarter(IScheduledTaskService scheduledTaskService) => TaskService = scheduledTaskService; public void Handle(ApplicationStartedEvent message) => TaskService?.InitializeTasks(); } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/SearchSniper/SearchSniperRepositoryHelper.cs ================================================ using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Qualities; namespace Tubifarry.Metadata.ScheduledTasks.SearchSniper { public sealed class SearchSniperRepositoryHelper( IMainDatabase database, IEventAggregator eventAggregator, IArtistService artistService) : BasicRepository(database, eventAggregator) { private readonly IArtistService _artistService = artistService; public List GetCutoffUnmetAlbumsBatch(Dictionary> profileCutoffs, int lastId, int limit) { if (profileCutoffs.Count == 0) return []; try { SqlBuilder builder = BuildCutoffUnmetQuery(profileCutoffs) .Where($@"""Albums"".""Id"" > {lastId}") .OrderBy($@"""Albums"".""Id"" ASC LIMIT {limit}"); return PopulateArtists(Query(builder)); } catch { return []; } } public List GetPartialAlbumsBatch(int lastId, int limit) { try { SqlBuilder builder = BuildMissingTracksQuery() .Where($@"""Albums"".""Id"" > {lastId}") .OrderBy($@"""Albums"".""Id"" ASC LIMIT {limit}"); return PopulateArtists(Query(builder)); } catch { return []; } } public (int minId, int maxId) GetPartialAlbumsIdRange() { try { SqlBuilder minBuilder = BuildMissingTracksQuery() .OrderBy($@"""Albums"".""Id"" ASC LIMIT 1"); List minResult = Query(minBuilder); if (minResult.Count == 0) return (0, 0); SqlBuilder maxBuilder = BuildMissingTracksQuery() .OrderBy($@"""Albums"".""Id"" DESC LIMIT 1"); List maxResult = Query(maxBuilder); return (minResult[0].Id, maxResult.Count > 0 ? maxResult[0].Id : minResult[0].Id); } catch { return (0, 0); } } public (int minId, int maxId) GetCutoffUnmetAlbumsIdRange(Dictionary> profileCutoffs) { if (profileCutoffs.Count == 0) return (0, 0); try { SqlBuilder minBuilder = BuildCutoffUnmetQuery(profileCutoffs) .OrderBy($@"""Albums"".""Id"" ASC LIMIT 1"); List minResult = Query(minBuilder); if (minResult.Count == 0) return (0, 0); SqlBuilder maxBuilder = BuildCutoffUnmetQuery(profileCutoffs) .OrderBy($@"""Albums"".""Id"" DESC LIMIT 1"); List maxResult = Query(maxBuilder); return (minResult[0].Id, maxResult.Count > 0 ? maxResult[0].Id : minResult[0].Id); } catch { return (0, 0); } } private SqlBuilder BuildCutoffUnmetQuery(Dictionary> profileCutoffs) { string whereClause = string.Join(" OR ", profileCutoffs.Select(kvp => $"(\"Artists\".\"QualityProfileId\" = {kvp.Key} AND " + $"CAST(json_extract(\"TrackFiles\".\"Quality\", '$.quality') AS INTEGER) IN ({string.Join(",", kvp.Value)}))" )); return Builder() .Join((a, ar) => a.ArtistMetadataId == ar.ArtistMetadataId) .Join((a, r) => a.Id == r.AlbumId) .Join((r, t) => r.Id == t.AlbumReleaseId) .Join((t, f) => t.TrackFileId == f.Id) .Where(a => a.Monitored == true) .Where(ar => ar.Monitored == true) .Where(r => r.Monitored == true) .Where($"({whereClause})") .GroupBy(x => x.Id); } private SqlBuilder BuildMissingTracksQuery() { return Builder() .Join((a, ar) => a.ArtistMetadataId == ar.ArtistMetadataId) .Join((a, r) => a.Id == r.AlbumId) .Join((r, t) => r.Id == t.AlbumReleaseId) .LeftJoin((t, f) => t.TrackFileId == f.Id) .Where(a => a.Monitored == true) .Where(ar => ar.Monitored == true) .Where(r => r.Monitored == true) .GroupBy(a => a.Id) .GroupBy(ar => ar.SortName) .Having("COUNT(DISTINCT \"Tracks\".\"Id\") > 0") .Having("SUM(CASE WHEN \"Tracks\".\"TrackFileId\" > 0 THEN 1 ELSE 0 END) > 0") .Having("SUM(CASE WHEN \"Tracks\".\"TrackFileId\" > 0 THEN 1 ELSE 0 END) < COUNT(DISTINCT \"Tracks\".\"Id\")"); } public static Dictionary> BuildProfileCutoffs(IEnumerable qualityProfiles) { Dictionary> profileCutoffs = []; foreach (QualityProfile profile in qualityProfiles) { if (!profile.UpgradeAllowed) continue; int cutoffIndex = profile.Items.FindIndex(x => (x.Quality?.Id == profile.Cutoff) || (x.Id == profile.Cutoff)); if (cutoffIndex <= 0) continue; List qualityIds = []; foreach (QualityProfileQualityItem item in profile.Items.Take(cutoffIndex)) { if (item.Quality != null) { qualityIds.Add(item.Quality.Id); } else if (item.Items?.Count > 0) { foreach (QualityProfileQualityItem groupItem in item.Items) { if (groupItem.Quality != null) qualityIds.Add(groupItem.Quality.Id); } } } if (qualityIds.Count != 0) profileCutoffs[profile.Id] = qualityIds; } return profileCutoffs; } private List PopulateArtists(List albums) { if (albums.Count == 0) return albums; HashSet neededMetadataIds = albums .Where(a => a.ArtistMetadataId > 0) .Select(a => a.ArtistMetadataId) .ToHashSet(); if (neededMetadataIds.Count == 0) return albums; Dictionary artistsByMetadataId = []; foreach (Artist artist in _artistService.GetAllArtists()) { if (neededMetadataIds.Contains(artist.ArtistMetadataId)) { artistsByMetadataId[artist.ArtistMetadataId] = artist; if (artistsByMetadataId.Count == neededMetadataIds.Count) break; } } foreach (Album album in albums) { if (artistsByMetadataId.TryGetValue(album.ArtistMetadataId, out Artist? artist)) album.Artist = new LazyLoaded(artist); } return albums; } public new SqlBuilder Builder() => base.Builder(); public new List Query(SqlBuilder builder) => base.Query(builder); } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/SearchSniper/SearchSniperTask.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.IndexerSearch; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Queue; using NzbDrone.Core.ThingiProvider; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.ScheduledTasks.SearchSniper { public class SearchSniperTask : ScheduledTaskBase, IExecute { private const int BatchSize = 100; private static readonly CacheService _cacheService = new(); private readonly IAlbumService _albumService; private readonly IArtistService _artistService; private readonly IQueueService _queueService; private readonly IManageCommandQueue _commandQueueManager; private readonly IQualityProfileService _qualityProfileService; private readonly SearchSniperRepositoryHelper _repositoryHelper; private readonly Logger _logger; public SearchSniperTask( IAlbumService albumService, IArtistService artistService, IQueueService queueService, IManageCommandQueue commandQueueManager, IQualityProfileService qualityProfileService, IMainDatabase database, IEventAggregator eventAggregator, Logger logger) { _albumService = albumService; _artistService = artistService; _queueService = queueService; _commandQueueManager = commandQueueManager; _qualityProfileService = qualityProfileService; _repositoryHelper = new SearchSniperRepositoryHelper(database, eventAggregator, artistService); _logger = logger; } public override string Name => "Search Sniper"; public override Type CommandType => typeof(SearchSniperCommand); public override ProviderMessage Message => new( "Automated search trigger that randomly selects albums for periodic scanning based on your search criteria. " + "Enable this metadata provider to start automatic searches.", ProviderMessageType.Info); private SearchSniperTaskSettings ActiveSettings => Settings ?? SearchSniperTaskSettings.Instance!; public override int IntervalMinutes => SearchSniperTaskSettings.Instance!.RefreshInterval; public override CommandPriority Priority => CommandPriority.Low; public override ValidationResult Test() { ValidationResult test = new(); InitializeCache(); if (ActiveSettings?.RequestCacheType == (int)CacheType.Permanent && !string.IsNullOrWhiteSpace(ActiveSettings.CacheDirectory) && !Directory.Exists(ActiveSettings.CacheDirectory)) { try { Directory.CreateDirectory(ActiveSettings.CacheDirectory); } catch (Exception ex) { test.Errors.Add(new ValidationFailure("CacheDirectory", $"Failed to create cache directory: {ex.Message}")); } } return test; } public void Execute(SearchSniperCommand message) { try { RunSearch(message); } catch (Exception ex) { _logger.Error(ex, "Error during scheduled execution"); } } private void InitializeCache() { if (ActiveSettings == null) return; _cacheService.CacheDuration = TimeSpan.FromDays(ActiveSettings.CacheRetentionDays); _cacheService.CacheType = (CacheType)ActiveSettings.RequestCacheType; _cacheService.CacheDirectory = ActiveSettings.CacheDirectory; } private void RunSearch(SearchSniperCommand message) { if (!ActiveSettings.SearchMissing && !ActiveSettings.SearchMissingTracks && !ActiveSettings.SearchQualityCutoffNotMet) { _logger.Warn("No search options enabled. Please enable at least one search criteria."); return; } if (ActiveSettings.StopWhenQueued > 0) { int queueCount = GetQueueCountByWaitOnType((WaitOnType)ActiveSettings.WaitOn); if (queueCount >= ActiveSettings.StopWhenQueued) { message.SetCompletionMessage($"Skipping Search Sniper, queue threshold reached ({queueCount} {(WaitOnType)ActiveSettings.WaitOn} items)"); _logger.Info("Skipping. Queue count ({0}) of {1} items reached threshold ({2})", queueCount, (WaitOnType)ActiveSettings.WaitOn, ActiveSettings.StopWhenQueued); return; } } int targetCount = ActiveSettings.RandomPicksPerInterval; HashSet queuedAlbumIds = GetQueuedAlbumIds(); int candidateTarget = Math.Min(targetCount * 10, 500); List eligibleAlbums = CollectEligibleAlbums(queuedAlbumIds, candidateTarget); if (eligibleAlbums.Count == 0) { message.SetCompletionMessage("Search Sniper completed. No eligible albums found."); _logger.Info("No eligible albums found after filtering queued and cached albums"); return; } List selectedAlbums = SelectRandomAlbums(eligibleAlbums, targetCount); foreach (Album album in selectedAlbums) _logger.Trace("Selected: '{0}' by {1}", album.Title, album.Artist?.Value?.Name ?? "Unknown Artist"); CacheSelectedAlbumsAsync(selectedAlbums).GetAwaiter().GetResult(); if (selectedAlbums.Count > 0) { _commandQueueManager.Push(new AlbumSearchCommand(selectedAlbums.ConvertAll(a => a.Id))); message.SetCompletionMessage($"Search Sniper completed. Queued {selectedAlbums.Count} album(s) for search"); _logger.Info("Queued {0} album(s) for search", selectedAlbums.Count); } } private List CollectEligibleAlbums(HashSet queuedAlbumIds, int candidateTarget) { Dictionary eligibleAlbums = []; Dictionary>? profileCutoffs = null; (int minId, int maxId) missingIdRange = (0, 0); (int minId, int maxId) partialIdRange = (0, 0); (int minId, int maxId) cutoffIdRange = (0, 0); if (ActiveSettings.SearchMissing) missingIdRange = GetMissingAlbumsIdRange(); if (ActiveSettings.SearchMissingTracks) partialIdRange = _repositoryHelper.GetPartialAlbumsIdRange(); if (ActiveSettings.SearchQualityCutoffNotMet) { profileCutoffs = SearchSniperRepositoryHelper.BuildProfileCutoffs(_qualityProfileService.All()); if (profileCutoffs.Count > 0) cutoffIdRange = _repositoryHelper.GetCutoffUnmetAlbumsIdRange(profileCutoffs); } if (ActiveSettings.SearchMissing && missingIdRange.maxId > 0 && eligibleAlbums.Count < candidateTarget) { int startId = GetRandomStartId(missingIdRange.minId, missingIdRange.maxId); _logger.Trace("Fetching missing albums (ID range: {0}-{1}, starting at ID: {2})...", missingIdRange.minId, missingIdRange.maxId, startId); CollectFromSource( lastId => GetMissingAlbumsBatch(lastId), eligibleAlbums, queuedAlbumIds, candidateTarget, startId, missingIdRange.minId); } if (ActiveSettings.SearchMissingTracks && partialIdRange.maxId > 0 && eligibleAlbums.Count < candidateTarget) { int startId = GetRandomStartId(partialIdRange.minId, partialIdRange.maxId); _logger.Trace("Fetching partial albums (ID range: {0}-{1}, starting at ID: {2})...", partialIdRange.minId, partialIdRange.maxId, startId); CollectFromSource( lastId => _repositoryHelper.GetPartialAlbumsBatch(lastId, BatchSize), eligibleAlbums, queuedAlbumIds, candidateTarget, startId, partialIdRange.minId); } if (ActiveSettings.SearchQualityCutoffNotMet && cutoffIdRange.maxId > 0 && eligibleAlbums.Count < candidateTarget) { int startId = GetRandomStartId(cutoffIdRange.minId, cutoffIdRange.maxId); _logger.Trace("Fetching cutoff unmet albums (ID range: {0}-{1}, starting at ID: {2})...", cutoffIdRange.minId, cutoffIdRange.maxId, startId); CollectFromSource( lastId => _repositoryHelper.GetCutoffUnmetAlbumsBatch(profileCutoffs!, lastId, BatchSize), eligibleAlbums, queuedAlbumIds, candidateTarget, startId, cutoffIdRange.minId); } AssignArtistsToAlbums(eligibleAlbums.Values); _logger.Info("Collected {0} eligible album(s) for random selection", eligibleAlbums.Count); return [.. eligibleAlbums.Values]; } private static int GetRandomStartId(int minId, int maxId) { if (maxId <= minId) return minId; return Random.Shared.Next(minId, maxId + 1); } private void CollectFromSource( Func> fetchBatch, Dictionary eligibleAlbums, HashSet queuedAlbumIds, int candidateTarget, int startId, int minId) { int lastId = startId - 1; bool hasWrapped = false; int maxIterations = 100; int iterations = 0; while (eligibleAlbums.Count < candidateTarget && iterations++ < maxIterations) { List batch = fetchBatch(lastId); if (batch.Count == 0) { if (!hasWrapped && lastId >= startId) { hasWrapped = true; lastId = minId - 1; continue; } break; } foreach (Album album in batch) { if (hasWrapped && album.Id >= startId) return; if (eligibleAlbums.Count >= candidateTarget) return; if (queuedAlbumIds.Contains(album.Id)) continue; if (eligibleAlbums.ContainsKey(album.Id)) continue; if (IsAlbumCached(album)) continue; eligibleAlbums[album.Id] = album; } lastId = batch[^1].Id; if (!hasWrapped && batch.Count < BatchSize) { hasWrapped = true; lastId = minId - 1; } } } private HashSet GetQueuedAlbumIds() => _queueService.GetQueue() .Where(q => q.Album is not null) .Select(q => q.Album!.Id) .ToHashSet(); private static bool IsAlbumCached(Album album) { string cacheKey = GenerateCacheKey(album); return _cacheService.GetAsync(cacheKey).GetAwaiter().GetResult(); } private void AssignArtistsToAlbums(IEnumerable albums) { List albumsNeedingArtist = albums.Where(a => a.Artist?.Value == null).ToList(); if (albumsNeedingArtist.Count == 0) return; HashSet artistIds = albumsNeedingArtist .Where(a => a.ArtistMetadataId > 0) .Select(a => a.ArtistMetadataId) .ToHashSet(); if (artistIds.Count == 0) return; Dictionary artistsByMetadataId = []; foreach (Artist artist in _artistService.GetAllArtists()) { if (artistIds.Contains(artist.ArtistMetadataId)) { artistsByMetadataId[artist.ArtistMetadataId] = artist; if (artistsByMetadataId.Count == artistIds.Count) break; } } foreach (Album album in albumsNeedingArtist) { if (artistsByMetadataId.TryGetValue(album.ArtistMetadataId, out Artist? artist)) album.Artist = new LazyLoaded(artist); } } private List GetMissingAlbumsBatch(int lastId) { PagingSpec pagingSpec = new() { Page = 1, PageSize = BatchSize, SortDirection = SortDirection.Ascending, SortKey = "Id" }; pagingSpec.FilterExpressions.Add(v => v.Id > lastId); pagingSpec.FilterExpressions.Add(v => v.Monitored == true && v.Artist.Value.Monitored == true); return _albumService.AlbumsWithoutFiles(pagingSpec).Records; } private (int minId, int maxId) GetMissingAlbumsIdRange() { try { PagingSpec minSpec = new() { Page = 1, PageSize = 1, SortDirection = SortDirection.Ascending, SortKey = "Id" }; minSpec.FilterExpressions.Add(v => v.Monitored == true && v.Artist.Value.Monitored == true); List minResult = _albumService.AlbumsWithoutFiles(minSpec).Records; if (minResult.Count == 0) return (0, 0); PagingSpec maxSpec = new() { Page = 1, PageSize = 1, SortDirection = SortDirection.Descending, SortKey = "Id" }; maxSpec.FilterExpressions.Add(v => v.Monitored == true && v.Artist.Value.Monitored == true); List maxResult = _albumService.AlbumsWithoutFiles(maxSpec).Records; return (minResult[0].Id, maxResult.Count > 0 ? maxResult[0].Id : minResult[0].Id); } catch (Exception ex) { _logger.Error(ex, "Error getting missing albums ID range"); return (0, 0); } } private int GetQueueCountByWaitOnType(WaitOnType waitOnType) { List queue = _queueService.GetQueue(); return waitOnType switch { WaitOnType.Queued => queue.Count(x => x.Status == "Queued"), WaitOnType.Downloading => queue.Count(x => x.Status == "Downloading"), WaitOnType.Warning => queue.Count(x => x.Status == "Warning"), WaitOnType.QueuedAndDownloading => queue.Count(x => x.Status == "Queued" || x.Status == "Downloading"), WaitOnType.All => queue.Count(x => x.Status != "Completed" && x.Status != "Failed"), _ => 0 }; } private static async Task CacheSelectedAlbumsAsync(List albums) { foreach (Album album in albums) { string cacheKey = GenerateCacheKey(album); await _cacheService.SetAsync(cacheKey, true); } } private static List SelectRandomAlbums(List albums, int count) { int pickCount = Math.Min(count, albums.Count); return albums.OrderBy(_ => Random.Shared.Next()).Take(pickCount).ToList(); } private static string GenerateCacheKey(Album album) => $"SearchSniper:{album.Artist?.Value?.Name ?? "Unknown"}:{album.Id}"; } } ================================================ FILE: Tubifarry/Metadata/ScheduledTasks/SearchSniper/SearchSniperTaskSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using Tubifarry.Core.Utilities; namespace Tubifarry.Metadata.ScheduledTasks.SearchSniper { public class SearchSniperSettingsValidator : AbstractValidator { public SearchSniperSettingsValidator() { // Validate RefreshInterval RuleFor(c => c.RefreshInterval) .GreaterThanOrEqualTo(5) .WithMessage("Refresh interval must be at least 5 minutes."); // When using Permanent cache, require a valid CacheDirectory RuleFor(x => x.CacheDirectory) .Must((settings, path) => (settings.RequestCacheType != (int)CacheType.Permanent) || (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path))) .WithMessage("A valid Cache Directory is required for Permanent caching."); // Validate CacheRetentionDays RuleFor(c => c.CacheRetentionDays) .GreaterThanOrEqualTo(1) .WithMessage("Retention time must be at least 1 day."); // Validate RandomPicksPerInterval RuleFor(c => c.RandomPicksPerInterval) .GreaterThanOrEqualTo(1) .WithMessage("At least 1 pick per interval is required."); // Validate StopWhenQueuedAlbumsReach RuleFor(c => c.StopWhenQueued) .GreaterThanOrEqualTo(0) .WithMessage("Stop queue threshold must be 0 or greater."); // Validatw search options RuleFor(c => c) .Must(settings => settings.SearchMissing || settings.SearchQualityCutoffNotMet || settings.SearchMissingTracks) .WithMessage("At least one search option must be enabled."); } } public class SearchSniperTaskSettings : IProviderConfig { protected static readonly AbstractValidator Validator = new SearchSniperSettingsValidator(); [FieldDefinition(1, Label = "Min Refresh Interval", Type = FieldType.Textbox, Unit = "minutes", Placeholder = "60", HelpText = "The minimum time between searches for random albums.")] public int RefreshInterval { get; set; } = 60; [FieldDefinition(2, Label = "Cache Directory", Type = FieldType.Path, Placeholder = "/config/cache", HelpText = "The directory where cached data will be stored. Leave empty for Memory cache.")] public string CacheDirectory { get; set; } = string.Empty; [FieldDefinition(3, Label = "Cache Retention Time", Type = FieldType.Number, Placeholder = "7", HelpText = "The number of days to retain cached data.")] public int CacheRetentionDays { get; set; } = 7; [FieldDefinition(4, Label = "Picks Per Interval", Type = FieldType.Number, Placeholder = "5", HelpText = "The number of random albums to search for during each refresh interval.")] public int RandomPicksPerInterval { get; set; } = 5; [FieldDefinition(5, Label = "Pause When Queued", Type = FieldType.Number, Placeholder = "0", HelpText = "Pause searching when the queue reaches this number. Set to 0 to disable.")] public int StopWhenQueued { get; set; } [FieldDefinition(6, Label = "Pause When Status", Type = FieldType.Select, SelectOptions = typeof(WaitOnType), HelpText = "Select which queue statuses should be counted when checking the 'Pause When Queued' threshold. This determines which types of items in the queue will prevent new searches from being triggered.")] public int WaitOn { get; set; } = (int)WaitOnType.QueuedAndDownloading; [FieldDefinition(7, Label = "Cache Type", Type = FieldType.Select, SelectOptions = typeof(CacheType), HelpText = "The type of cache to use for storing search results. Memory cache is faster but does not persist after restart. Permanent cache persists on disk but requires a valid directory.")] public int RequestCacheType { get; set; } = (int)CacheType.Memory; [FieldDefinition(8, Label = "Missing", Type = FieldType.Checkbox, HelpText = "Search for albums that are missing from your library.")] public bool SearchMissing { get; set; } = true; [FieldDefinition(9, Label = "Missing Tracks", Type = FieldType.Checkbox, HelpText = "Automatically search for albums that have missing tracks in your library.")] public bool SearchMissingTracks { get; set; } [FieldDefinition(10, Label = "Cutoff Not Met", Type = FieldType.Checkbox, HelpText = "Automatically search for albums where the current quality does not meet the quality cutoff.")] public bool SearchQualityCutoffNotMet { get; set; } public string BaseUrl { get; set; } = string.Empty; public SearchSniperTaskSettings() => Instance = this; public static SearchSniperTaskSettings? Instance { get; private set; } public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum WaitOnType { [FieldOption(Label = "Queued Only", Hint = "Count only items waiting to start downloading")] Queued = 0, [FieldOption(Label = "Downloading Only", Hint = "Count only items actively downloading")] Downloading = 1, [FieldOption(Label = "Warning Only", Hint = "Count only items with warnings")] Warning = 2, [FieldOption(Label = "Queued + Downloading", Hint = "Count items that are queued or actively downloading")] QueuedAndDownloading = 3, [FieldOption(Label = "All Active Items", Hint = "Count all non-completed items")] All = 4 } public class SearchSniperCommand : Command { public override bool SendUpdatesToClient => true; public override bool UpdateScheduledTask => true; public override string CompletionMessage => _completionMessage ?? "Search Sniper completed"; private string? _completionMessage; public void SetCompletionMessage(string message) => _completionMessage = message; } } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareDetector.cs ================================================ using NLog; using NzbDrone.Common.Instrumentation; using System.Net; namespace Tubifarry.Notifications.FlareSolverr; /// /// Detects Cloudflare and DDoS-GUARD protection challenges in HTTP responses /// public static class FlareDetector { private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(FlareDetector)); private static readonly string[] CloudflareServerNames = ["cloudflare", "cloudflare-nginx", "ddos-guard"]; private static readonly string[] CloudflareCookiePrefixes = ["cf_", "__cf", "__ddg"]; private static readonly string[] CloudflareChallengeIndicators = [ "Just a moment...", "Checking your browser", "jschl-answer", // JavaScript challenge form field "cf-challenge", // Cloudflare challenge class "cf-chl-bypass", // Challenge bypass indicator "Verify you are human", // Interactive challenge button "DDOS-GUARD" ]; private static readonly string[] CloudflareBlockIndicators = [ "

Sorry, you have been blocked

", "

Sorry, you have been blocked

", "You are unable to access", "This website is using a security service to protect itself" ]; private static readonly HttpStatusCode[] ProtectionStatusCodes = [ HttpStatusCode.ServiceUnavailable, // 503 HttpStatusCode.Forbidden, // 403 (HttpStatusCode)523 // Cloudflare: Origin Unreachable ]; /// /// Checks if the HTTP response indicates Cloudflare protection is active /// public static bool IsChallengePresent(HttpResponseMessage response) { ArgumentNullException.ThrowIfNull(response); string url = response.RequestMessage?.RequestUri?.ToString() ?? "(unknown)"; // Check status codes, successful responses (200-299) are never challenges if (response.IsSuccessStatusCode) { Logger.Trace("Status code {0} is successful, no challenge for {1}", response.StatusCode, url); return false; } // Check status codes that indicate protection challenges if (!ProtectionStatusCodes.Contains(response.StatusCode)) { Logger.Trace("Status code {0} not a protection status code for {1}", response.StatusCode, url); return false; } // Check if Server header indicates Cloudflare/DDoS-GUARD bool isCloudflareServer = response.Headers.Server.Any(server => server.Product != null && CloudflareServerNames.Contains(server.Product.Name.ToLowerInvariant())); if (isCloudflareServer) { string? serverName = response.Headers.Server.First(s => s.Product != null).Product?.Name; Logger.Trace("Cloudflare/DDoS-Guard server detected ({0}) for {1}", serverName, url); } else { Logger.Trace("Not a Cloudflare/DDoS-Guard server for {0}", url); return false; } string responseText = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); string? blockIndicator = CloudflareBlockIndicators.FirstOrDefault(indicator => responseText.Contains(indicator, StringComparison.OrdinalIgnoreCase)); if (blockIndicator != null) { Logger.Trace("Cloudflare block detected (not a challenge) for {0}. Indicator: '{1}'. FlareSolverr cannot bypass IP/region blocks.", url, blockIndicator.Substring(0, Math.Min(50, blockIndicator.Length))); return false; } // Check for Cloudflare error codes if (responseText.TrimStart().StartsWith("error code:", StringComparison.OrdinalIgnoreCase)) { Logger.Trace("Cloudflare error code detected in content for {0}", url); return true; } // Check for actual challenge indicators string? challengeIndicator = CloudflareChallengeIndicators.FirstOrDefault(indicator => responseText.Contains(indicator, StringComparison.OrdinalIgnoreCase)); if (challengeIndicator != null) { Logger.Trace("Challenge indicator '{0}' found in content for {1}", challengeIndicator, url); return true; } // Check for custom Cloudflare configurations (some Dutch torrent sites) if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" && string.IsNullOrEmpty(response.Content.Headers.ContentEncoding.ToString()) && responseText.Contains("ddos", StringComparison.OrdinalIgnoreCase)) { Logger.Trace("Custom DDoS protection pattern detected for {0}", url); return true; } Logger.Trace("No protection challenge detected for {0}", url); return false; } /// /// Checks if a cookie name is a Cloudflare/DDoS-GUARD cookie /// public static bool IsProtectionCookie(string cookieName) => CloudflareCookiePrefixes.Any(prefix => cookieName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareRecords.cs ================================================ using System.Net; using System.Text.Json.Serialization; namespace Tubifarry.Notifications.FlareSolverr; /// /// Represents a cookie returned from FlareSolverr /// public record FlareCookie( [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("value")] string Value, [property: JsonPropertyName("domain")] string Domain, [property: JsonPropertyName("path")] string Path = "/", [property: JsonPropertyName("expires")] int Expiry = 0, [property: JsonPropertyName("httpOnly")] bool HttpOnly = false, [property: JsonPropertyName("secure")] bool Secure = false, [property: JsonPropertyName("sameSite")] string? SameSite = null) { /// /// Converts to System.Net.Cookie /// public Cookie ToCookie() => new(Name, Value, Path, Domain) { HttpOnly = HttpOnly, Secure = Secure }; /// /// Converts to Cookie header value /// public string ToHeaderValue() => $"{Name}={Value}"; } /// /// Base request for FlareSolverr API /// public record FlareRequest( [property: JsonPropertyName("cmd")] string Command, [property: JsonPropertyName("url")] string? Url = null, [property: JsonPropertyName("maxTimeout")] int MaxTimeout = 60000); /// /// Solution data from FlareSolverr /// public record FlareSolution( [property: JsonPropertyName("url")] string Url, [property: JsonPropertyName("status")] int Status, [property: JsonPropertyName("cookies")] FlareCookie[] Cookies, [property: JsonPropertyName("userAgent")] string? UserAgent = null, [property: JsonPropertyName("response")] string? Response = null) { /// /// Checks if the solution contains valid cookies /// public bool HasValidCookies => Cookies?.Length > 0; } /// /// Response from FlareSolverr API /// public record FlareResponse( [property: JsonPropertyName("status")] string Status, [property: JsonPropertyName("message")] string Message, [property: JsonPropertyName("solution")] FlareSolution? Solution = null, [property: JsonPropertyName("startTimestamp")] long StartTimestamp = 0, [property: JsonPropertyName("endTimestamp")] long EndTimestamp = 0, [property: JsonPropertyName("version")] string? Version = null) { /// /// Checks if the request was successful /// public bool IsSuccess => Status.Equals("ok", StringComparison.OrdinalIgnoreCase); /// /// Gets the duration of the request in milliseconds /// public long DurationMs => EndTimestamp - StartTimestamp; } /// /// FlareSolverr index/health check response /// public record FlareIndexResponse( [property: JsonPropertyName("msg")] string Message, [property: JsonPropertyName("version")] string? Version = null, [property: JsonPropertyName("userAgent")] string? UserAgent = null); ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareSolverrHttpInterceptor.cs ================================================ using NLog; using NzbDrone.Common.Http; namespace Tubifarry.Notifications.FlareSolverr { public class FlareSolverrHttpInterceptor( Logger logger, Lazy flareService, Lazy httpClient) : IHttpRequestInterceptor { private readonly Logger _logger = logger; private readonly Lazy _flareService = flareService; private readonly Lazy _httpClient = httpClient; private IFlareSolverrService? FlareService { get { try { return _flareService.Value; } catch { return null; } } } public HttpRequest PreRequest(HttpRequest request) { if (FlareService?.IsEnabled != true) { return request; } string host = request.Url.Host; if (FlareService.HasValidSolution(host)) { ProtectionSolution? solution = FlareService.GetOrSolveChallenge(host, request.Url.ToString()); if (solution != null) ApplySolution(request, solution); } return request; } public HttpResponse PostResponse(HttpResponse response) { if (FlareService?.IsEnabled != true || !IsProtectionChallengeDetected(response)) { return response; } string host = response.Request.Url.Host; int retryCount = GetRetryCount(response.Request); int maxRetries = FlareService.MaxRetries; if (retryCount >= maxRetries) { _logger.Error("Max retries ({0}) reached for {1}", maxRetries, host); return response; } _logger.Warn("Protection challenge detected for {0} (attempt {1}/{2}) - Status: {3}", host, retryCount + 1, maxRetries, response.StatusCode); _logger.Trace("Current request has {0} cookies", response.Request.Cookies.Count); string baseUrl = $"{response.Request.Url.Scheme}://{response.Request.Url.Host}/"; ProtectionSolution? solution = FlareService.GetOrSolveChallenge(host, baseUrl, forceNew: true); HttpRequest retryRequest = CloneRequest(response.Request); SetRetryCount(retryRequest, retryCount + 1); if (solution == null) { _logger.Error("Failed to solve challenge for {0}, retrying anyway (attempt {1}/{2})", host, retryCount + 1, maxRetries); return _httpClient.Value.Execute(retryRequest); } _logger.Info("Challenge solved for {0}, retrying with {1} cookies: [{2}]", host, solution.Cookies.Length, string.Join(", ", solution.Cookies.Select(c => c.Name))); ApplySolution(retryRequest, solution); return _httpClient.Value.Execute(retryRequest); } private void ApplySolution(HttpRequest request, ProtectionSolution solution) { if (!string.IsNullOrWhiteSpace(solution.UserAgent)) request.Headers["User-Agent"] = solution.UserAgent; foreach (FlareCookie cookie in solution.Cookies) { _logger.Trace("Applying cookie: {0} = {1}... (domain: {2}, protection: {3})", cookie.Name, cookie.Value[..Math.Min(20, cookie.Value.Length)], cookie.Domain, FlareDetector.IsProtectionCookie(cookie.Name)); request.Cookies[cookie.Name] = cookie.Value; } } private static HttpRequest CloneRequest(HttpRequest original) { HttpRequest clone = new(original.Url.ToString()) { Method = original.Method, AllowAutoRedirect = original.AllowAutoRedirect, ContentData = original.ContentData, LogHttpError = original.LogHttpError, LogResponseContent = original.LogResponseContent, RateLimit = original.RateLimit, RateLimitKey = original.RateLimitKey, RequestTimeout = original.RequestTimeout, StoreRequestCookie = original.StoreRequestCookie, StoreResponseCookie = original.StoreResponseCookie, SuppressHttpError = original.SuppressHttpError, SuppressHttpErrorStatusCodes = original.SuppressHttpErrorStatusCodes }; foreach (KeyValuePair header in original.Headers) clone.Headers[header.Key] = header.Value; foreach (KeyValuePair cookie in original.Cookies) clone.Cookies[cookie.Key] = cookie.Value; return clone; } private static bool IsProtectionChallengeDetected(HttpResponse response) { try { string content = response.Content ?? string.Empty; HttpResponseMessage httpResponse = new(response.StatusCode) { Content = new StringContent(content) }; if (response.Headers != null) { foreach (KeyValuePair header in response.Headers) { try { if (header.Key.Equals("Server", StringComparison.OrdinalIgnoreCase)) { httpResponse.Headers.TryAddWithoutValidation(header.Key, header.Value); } } catch { } } } bool isChallenge = FlareDetector.IsChallengePresent(httpResponse); httpResponse.Dispose(); return isChallenge; } catch (Exception) { return false; } } private static int GetRetryCount(HttpRequest request) => request.Cookies.TryGetValue("_flare_retry", out string? value) && int.TryParse(value, out int count) ? count : 0; private static void SetRetryCount(HttpRequest request, int count) => request.Cookies["_flare_retry"] = count.ToString(); } } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareSolverrNotification.cs ================================================ using FluentValidation.Results; using NzbDrone.Core.Notifications; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Notifications.FlareSolverr { public class FlareSolverrNotification : NotificationBase { public override string Name => "FlareSolverr"; public override string Link => "https://github.com/FlareSolverr/FlareSolverr"; public override ProviderMessage Message => new( "FlareSolverr automatically bypasses Cloudflare and DDoS-GUARD protection challenges for HTTP requests. " + "Requires a running FlareSolverr instance (Docker recommended). " + "Configure the API URL to enable transparent challenge solving.", ProviderMessageType.Info); public override ValidationResult Test() => new(); public override void OnGrab(GrabMessage message) { } } } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareSolverrService.cs ================================================ using DownloadAssistant.Base; using NLog; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Notifications; using System.Net; using System.Text.Json; using System.Text.Json.Serialization; using Tubifarry.Core.Utilities; namespace Tubifarry.Notifications.FlareSolverr { public class FlareSolverrService : IFlareSolverrService, IHandle { private readonly INotificationFactory _notificationFactory; private readonly INotificationStatusService _notificationStatusService; private readonly Logger _logger; private readonly CacheService _cache; private string? _apiUrl; private int _maxRetries = 3; private int _maxTimeout = 60000; private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, WriteIndented = false }; public FlareSolverrService( INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger) { _notificationFactory = notificationFactory; _notificationStatusService = notificationStatusService; _logger = logger; _cache = new() { CacheType = CacheType.Memory }; } public bool IsEnabled => !string.IsNullOrEmpty(_apiUrl); public int MaxRetries => _maxRetries; public void Handle(ApplicationStartedEvent message) { try { ConfigureFlareSolverr(); } catch (Exception ex) { _logger.Error(ex, "Failed to configure FlareSolverr on application startup"); } } public ProtectionSolution? GetOrSolveChallenge(string host, string url, bool forceNew = false) { if (!IsEnabled) { _logger.Error("FlareSolverr is not configured"); return null; } string cacheKey = $"flare_solution_{host}"; if (!forceNew) { ProtectionSolution? cached = _cache.GetAsync(cacheKey).GetAwaiter().GetResult(); if (cached?.IsValid == true) { _logger.Trace("Using cached solution for {0}, expires in {1:0.0} minutes", host, cached.TimeUntilExpiry.TotalMinutes); return cached; } } _logger.Info("Solving challenge for {0} using URL: {1}", host, url); return SolveChallenge(host, url, cacheKey); } public void InvalidateSolution(string host) { string cacheKey = $"flare_solution_{host}"; _cache.SetAsync(cacheKey, (ProtectionSolution?)null).GetAwaiter().GetResult(); _logger.Debug("Invalidated solution for {0}", host); } public bool HasValidSolution(string host) { string cacheKey = $"flare_solution_{host}"; ProtectionSolution? solution = _cache.GetAsync(cacheKey).GetAwaiter().GetResult(); return solution?.IsValid == true; } private ProtectionSolution? SolveChallenge(string host, string url, string cacheKey) { try { FlareResponse? result = SolveAsync(url).GetAwaiter().GetResult(); if (result?.IsSuccess != true || result.Solution?.HasValidCookies != true) { _logger.Warn("Challenge solve unsuccessful for {0}. Status: {1}, Cookies: {2}", host, result?.Status ?? "null", result?.Solution?.Cookies?.Length ?? 0); return null; } DateTime expiry = DateTime.UtcNow.Add(_cache.CacheDuration); ProtectionSolution solution = new( result.Solution.Cookies, result.Solution.UserAgent, expiry, host); _cache.SetAsync(cacheKey, solution).GetAwaiter().GetResult(); _logger.Info("Successfully solved challenge for {0}, cached until {1:yyyy-MM-dd HH:mm:ss} UTC. Cookies: {2}", host, expiry, solution.Cookies.Length); return solution; } catch (FlareException ex) { _logger.Error(ex, "Exception while solving challenge for {0}: {1}", host, ex.Message); return null; } catch (Exception ex) { _logger.Error(ex, "Unexpected exception while solving challenge for {0}", host); return null; } } private async Task SolveAsync(string url) { if (string.IsNullOrEmpty(_apiUrl)) throw new InvalidOperationException("FlareSolverr API URL is not configured"); Uri apiEndpoint = new($"{_apiUrl.TrimEnd('/')}/v1"); FlareRequest request = new( Command: "request.get", Url: url, MaxTimeout: _maxTimeout ); try { string json = JsonSerializer.Serialize(request, JsonOptions); _logger.Trace("Sending request to {0}", apiEndpoint); StringContent content = new(json, System.Text.Encoding.UTF8, "application/json"); HttpResponseMessage response = await HttpGet.HttpClient.PostAsync(apiEndpoint, content); _logger.Trace("Received response with status code: {0}", response.StatusCode); if (response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.InternalServerError) { string errorBody = await response.Content.ReadAsStringAsync(); _logger.Error("Unexpected HTTP status {0}", response.StatusCode); throw new FlareException($"Unexpected status code from FlareSolverr: {response.StatusCode}"); } string responseText = await response.Content.ReadAsStringAsync(); FlareResponse? flareResponse = JsonSerializer.Deserialize(responseText, JsonOptions) ?? throw new FlareException("FlareSolverr returned null response"); if (!flareResponse.IsSuccess) { string errorMessage = flareResponse.Status.ToLowerInvariant() switch { "warning" => $"Captcha detected: {flareResponse.Message}", "error" => $"FlareSolverr error: {flareResponse.Message}", _ => $"Unknown status '{flareResponse.Status}': {flareResponse.Message}" }; _logger.Warn("Request failed - {0}", errorMessage); throw new FlareException(errorMessage); } if (flareResponse.Solution != null) { _logger.Trace("Received response. Duration: {0}ms, Cookies: {1}", flareResponse.DurationMs, flareResponse.Solution.Cookies?.Length ?? 0); if (flareResponse.Solution.Cookies?.Length > 0) { _logger.Trace("Received cookies: {0}", string.Join(", ", flareResponse.Solution.Cookies.Select(c => c.Name))); } } return flareResponse; } catch (HttpRequestException ex) { _logger.Error(ex, "Failed to connect to FlareSolverr API at {0}", apiEndpoint); throw new FlareException($"Failed to connect to FlareSolverr: {ex.Message}", ex); } catch (JsonException ex) { _logger.Error(ex, "Failed to parse JSON response"); throw new FlareException($"Failed to parse FlareSolverr response: {ex.Message}", ex); } } private void ConfigureFlareSolverr() { foreach (INotification? notification in (List)_notificationFactory.GetAvailableProviders()) { if (notification is not FlareSolverrNotification) continue; if (notification.Definition is not NotificationDefinition definition || !definition.Enable) continue; FlareSolverrSettings settings = (FlareSolverrSettings)notification.Definition.Settings; try { _apiUrl = settings.ApiUrl; _maxRetries = settings.MaxRetries; _maxTimeout = settings.MaxTimeout; _cache.CacheDuration = TimeSpan.FromMinutes(settings.CacheDurationMinutes); HttpGet.HttpClient.Timeout = TimeSpan.FromMilliseconds(_maxTimeout + 5000); TestConnection(); _notificationStatusService.RecordSuccess(notification.Definition.Id); _logger.Trace("Successfully configured FlareSolverr: {0}", settings.ApiUrl); } catch (Exception ex) { _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Error(ex, "Failed to configure FlareSolverr with settings"); } } } private void TestConnection() { if (string.IsNullOrEmpty(_apiUrl)) return; try { Uri indexEndpoint = new(_apiUrl.TrimEnd('/') + "/"); HttpResponseMessage response = HttpGet.HttpClient.GetAsync(indexEndpoint).GetAwaiter().GetResult(); if (response.IsSuccessStatusCode) { string content = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); FlareIndexResponse? indexResponse = JsonSerializer.Deserialize(content, JsonOptions); if (indexResponse != null) { _logger.Debug("FlareSolverr connection successful. Version: {0}", indexResponse.Version ?? "Unknown"); } } } catch (Exception ex) { _logger.Warn(ex, "Could not verify FlareSolverr connection, but will proceed with configuration"); } } } public class FlareException : Exception { public FlareException(string message) : base(message) { } public FlareException(string message, Exception innerException) : base(message, innerException) { } public FlareException() { } } } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/FlareSolverrSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Notifications.FlareSolverr { public class FlareSolverrSettingsValidator : AbstractValidator { public FlareSolverrSettingsValidator() { RuleFor(c => c.ApiUrl) .NotEmpty() .WithMessage("FlareSolverr API URL is required") .Must(url => Uri.IsWellFormedUriString(url, UriKind.Absolute)) .WithMessage("FlareSolverr API URL must be a valid absolute URL"); RuleFor(c => c.MaxTimeout) .GreaterThan(0) .LessThanOrEqualTo(300000) .WithMessage("Max timeout must be between 1 and 300000 milliseconds"); RuleFor(c => c.CacheDurationMinutes) .GreaterThanOrEqualTo(1) .LessThanOrEqualTo(1440) .WithMessage("Cache duration must be between 1min and 24 hours"); RuleFor(c => c.MaxRetries) .GreaterThan(0) .LessThanOrEqualTo(10) .WithMessage("Max retries must be between 1 and 10"); } } public class FlareSolverrSettings : IProviderConfig { private static readonly FlareSolverrSettingsValidator Validator = new(); [FieldDefinition(1, Label = "FlareSolverr API URL", Type = FieldType.Textbox, HelpText = "The URL of your FlareSolverr instance (e.g., http://localhost:8191/).")] public string ApiUrl { get; set; } = "http://localhost:8191/"; [FieldDefinition(2, Label = "Max Timeout (ms)", Type = FieldType.Number, HelpText = "Maximum timeout in milliseconds for solving challenges (default: 60000 = 60 seconds).")] public int MaxTimeout { get; set; } = 60000; [FieldDefinition(3, Label = "Cache Duration (minutes)", Type = FieldType.Number, HelpText = "How long to cache solved challenges in memory (default: 30 minutes).", Advanced = true)] public int CacheDurationMinutes { get; set; } = 30; [FieldDefinition(4, Label = "Max Retries", Type = FieldType.Number, HelpText = "Maximum number of times to retry solving a challenge (default: 3).", Advanced = true)] public int MaxRetries { get; set; } = 3; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Notifications/FlareSolverr/IFlareSolverrService.cs ================================================ namespace Tubifarry.Notifications.FlareSolverr { public interface IFlareSolverrService { bool IsEnabled { get; } int MaxRetries { get; } ProtectionSolution? GetOrSolveChallenge(string host, string url, bool forceNew = false); void InvalidateSolution(string host); bool HasValidSolution(string host); } public record ProtectionSolution( FlareCookie[] Cookies, string? UserAgent, DateTime ExpiryUtc, string Host) { public bool IsValid => DateTime.UtcNow < ExpiryUtc && Cookies.Length > 0; public TimeSpan TimeUntilExpiry => ExpiryUtc - DateTime.UtcNow; } } ================================================ FILE: Tubifarry/Notifications/PlaylistExport/PlaylistExportNotification.cs ================================================ using FluentValidation.Results; using NzbDrone.Core.Notifications; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Notifications.PlaylistExport; public sealed class PlaylistExportNotification : NotificationBase { private readonly IPlaylistExportService _service; public PlaylistExportNotification(IPlaylistExportService service) => _service = service; public override string Name => "Playlist Export"; public override string Link => "https://github.com/TypNull/Tubifarry"; public override ProviderMessage Message => new("Generates .m3u8 playlist files for selected import lists whenever a track is imported.", ProviderMessageType.Info); public override void OnReleaseImport(AlbumDownloadMessage message) => _service.GeneratePlaylists(Settings); public override ValidationResult Test() => new(); } ================================================ FILE: Tubifarry/Notifications/PlaylistExport/PlaylistExportService.cs ================================================ using Lidarr.Http.ClientSchema; using NLog; using NzbDrone.Core.ImportLists; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Notifications; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Events; using System.Text; using System.Text.RegularExpressions; using Tubifarry.Core.Model; using Tubifarry.Core.Utilities; namespace Tubifarry.Notifications.PlaylistExport; public interface IPlaylistExportService { void RefreshSchema(); void FetchAndStore(int listId); void GeneratePlaylists(PlaylistExportSettings settings); string? DetectCommonMusicPath(); } public sealed partial class PlaylistExportService : IPlaylistExportService, IHandle, IHandle>, IHandle>, IHandle> { private const string SnapshotKey = "playlistExport.snapshots"; private readonly IImportListFactory _importListFactory; private readonly IFetchAndParseImportList _fetchAndParse; private readonly IPluginSettings _pluginSettings; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly ITrackService _trackService; private readonly IMediaFileService _mediaFileService; private readonly Lazy _notificationFactory; private readonly Logger _logger; public PlaylistExportService( IImportListFactory importListFactory, IFetchAndParseImportList fetchAndParse, IPluginSettings pluginSettings, IArtistService artistService, IAlbumService albumService, ITrackService trackService, IMediaFileService mediaFileService, Lazy notificationFactory, Logger logger) { _importListFactory = importListFactory; _fetchAndParse = fetchAndParse; _pluginSettings = pluginSettings; _artistService = artistService; _albumService = albumService; _trackService = trackService; _mediaFileService = mediaFileService; _notificationFactory = notificationFactory; _logger = logger; } public void Handle(ApplicationStartedEvent message) => RefreshSchema(); public void Handle(ProviderAddedEvent message) => RefreshSchema(); public void Handle(ProviderUpdatedEvent message) => RefreshSchema(); public void Handle(ProviderDeletedEvent message) { Dictionary snapshots = GetSnapshots(); if (snapshots.Remove(message.ProviderId, out PlaylistSnapshot? deleted)) { SaveSnapshots(snapshots); foreach (INotification n in _notificationFactory.Value.GetAvailableProviders() .OfType()) { PlaylistExportSettings s = (PlaylistExportSettings)n.Definition.Settings; if (!s.CleanupOnRemove || string.IsNullOrEmpty(s.OutputPath)) continue; string m3u8Path = Path.Combine(s.OutputPath, $"{SanitizeFilename(deleted.ListName)}.m3u8"); if (File.Exists(m3u8Path)) { _logger.Debug($"Deleting {m3u8Path} (import list removed)"); File.Delete(m3u8Path); } } } RefreshSchema(); } public void RefreshSchema() { List allLists = _importListFactory.GetAvailableProviders(); int order = 6; List dynamicMappings = []; foreach (IImportList l in allLists) { string key = $"list_{l.Definition.Id}"; dynamicMappings.Add(new FieldMapping { Field = new Field { Name = key, Label = l.Definition.Name, Type = "checkbox", HelpText = l is IPlaylistTrackSource ? $"Supports track-level data, generates a track-specific playlist for '{l.Definition.Name}'" : $"Album-level only, generates a playlist of all local tracks for '{l.Definition.Name}'", Order = order++, }, PropertyType = typeof(bool), GetterFunc = m => ((PlaylistExportSettings)m).GetBoolState(key), SetterFunc = (m, v) => ((PlaylistExportSettings)m).SetBoolState(key, Convert.ToBoolean(v)), }); } DynamicSchemaInjector.InjectDynamic(dynamicMappings, "list_"); _logger.Debug($"Schema refreshed with {allLists.Count} import list(s)"); } public string? DetectCommonMusicPath() { List paths = _artistService.GetAllArtists() .Select(a => a.Path) .Where(p => !string.IsNullOrEmpty(p)) .ToList(); if (paths.Count == 0) return null; return FindCommonRoot(paths); } public void FetchAndStore(int listId) { IImportList? list = _importListFactory.GetAvailableProviders() .FirstOrDefault(l => l.Definition.Id == listId); if (list == null) { _logger.Warn($"Import list ID {listId} not found"); return; } _logger.Debug($"Fetching items from '{list.Definition.Name}'"); List items = list is IPlaylistTrackSource trackSource ? trackSource.FetchTrackLevelItems() : FetchAlbumLevelItems(list); Dictionary snapshots = GetSnapshots(); snapshots[listId] = new PlaylistSnapshot(list.Definition.Name, items, DateTime.UtcNow); SaveSnapshots(snapshots); _logger.Info($"Stored {items.Count} item(s) for '{list.Definition.Name}'"); } public void GeneratePlaylists(PlaylistExportSettings settings) { string? outputPath = settings.AutoDetectOutputPath ? DetectCommonMusicPath() : settings.OutputPath; if (string.IsNullOrWhiteSpace(outputPath)) { _logger.Warn("OutputPath not configured and auto-detect found nothing, skipping generation"); return; } List selectedIds = settings.GetSelectedListIds().ToList(); if (selectedIds.Count == 0) return; Dictionary snapshots = GetSnapshots(); PlaylistTrackMode trackMode = settings.GetTrackMode(); List allLists = _importListFactory.GetAvailableProviders(); HashSet trackSourceIds = allLists .Where(l => l is IPlaylistTrackSource) .Select(l => l.Definition.Id) .ToHashSet(); List allArtists = _artistService.GetAllArtists(); Dictionary artistByMbId = allArtists .ToDictionary(a => a.ForeignArtistId, StringComparer.OrdinalIgnoreCase); Dictionary artistByName = allArtists .GroupBy(a => Normalize(a.Name)) .ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); Dictionary albumByMbId = _albumService.GetAllAlbums() .ToDictionary(a => a.ForeignAlbumId, StringComparer.OrdinalIgnoreCase); Directory.CreateDirectory(outputPath); foreach (int listId in selectedIds) { if (trackMode == PlaylistTrackMode.TrackDataOnly && !trackSourceIds.Contains(listId)) { _logger.Debug($"Skipping list {listId}: TrackDataOnly mode and list does not support track-level data"); continue; } if (!snapshots.TryGetValue(listId, out PlaylistSnapshot? snapshot)) { _logger.Warn($"No snapshot for list {listId}: fetch has not run yet for this list."); continue; } List files = []; foreach (PlaylistItem item in snapshot.Items) { bool useTrackLevel = trackMode != PlaylistTrackMode.AlbumDataOnly && (item.TrackTitle != null || item.ForeignRecordingId != null); if (!useTrackLevel) { files.AddRange(GetAlbumOrArtistFiles(item, albumByMbId, artistByMbId, artistByName)); continue; } Artist? artist = ResolveArtist(item, artistByMbId, artistByName); if (artist == null) continue; if (item.ForeignRecordingId != null) { Track? t = _trackService.GetTracksByArtist(artist.Id) .FirstOrDefault(t => string.Equals(t.ForeignRecordingId, item.ForeignRecordingId, StringComparison.OrdinalIgnoreCase) && t.TrackFileId > 0); if (t != null) files.Add(_mediaFileService.Get(t.TrackFileId)); } else if (item.TrackTitle != null) { IEnumerable candidates = item.AlbumMusicBrainzId != null && albumByMbId.TryGetValue(item.AlbumMusicBrainzId, out Album? alb) ? _trackService.GetTracksByAlbum(alb.Id) : _trackService.GetTracksByArtist(artist.Id); Track? t = candidates.FirstOrDefault(t => Normalize(t.Title) == Normalize(item.TrackTitle) && t.TrackFileId > 0); if (t != null) files.Add(_mediaFileService.Get(t.TrackFileId)); } } WriteM3u8(outputPath, snapshot.ListName, files, settings.UseRelativePaths); } } private List FetchAlbumLevelItems(IImportList list) { List raw = _fetchAndParse.FetchSingleList((ImportListDefinition)list.Definition); return raw .Where(i => !string.IsNullOrEmpty(i.ArtistMusicBrainzId)) .Select(i => new PlaylistItem( i.ArtistMusicBrainzId, string.IsNullOrEmpty(i.AlbumMusicBrainzId) ? null : i.AlbumMusicBrainzId, i.Artist ?? "", string.IsNullOrEmpty(i.Album) ? null : i.Album)) .DistinctBy(i => (i.ArtistMusicBrainzId, i.AlbumMusicBrainzId)) .ToList(); } private IEnumerable GetAlbumOrArtistFiles( PlaylistItem item, Dictionary albumByMbId, Dictionary artistByMbId, Dictionary artistByName) { if (item.AlbumMusicBrainzId != null && albumByMbId.TryGetValue(item.AlbumMusicBrainzId, out Album? album)) { return _mediaFileService.GetFilesByAlbum(album.Id); } Artist? artist = ResolveArtist(item, artistByMbId, artistByName); if (artist != null) return _mediaFileService.GetFilesByArtist(artist.Id); return []; } private static Artist? ResolveArtist( PlaylistItem item, Dictionary byMbId, Dictionary byName) { if (!string.IsNullOrEmpty(item.ArtistMusicBrainzId) && byMbId.TryGetValue(item.ArtistMusicBrainzId, out Artist? a)) return a; string key = Normalize(item.ArtistName); return string.IsNullOrEmpty(key) ? null : byName.GetValueOrDefault(key); } private static string Normalize(string? s) => s == null ? "" : NormalizeRegex().Replace(s.ToLowerInvariant(), ""); private void WriteM3u8(string outputPath, string listName, List files, bool useRelative) { string filename = SanitizeFilename(listName) + ".m3u8"; string fullPath = Path.Combine(outputPath, filename); List lines = ["#EXTM3U", $"#PLAYLIST:{listName}"]; foreach (TrackFile tf in files.Where(f => File.Exists(f.Path))) { string displayName = Path.GetFileNameWithoutExtension(tf.Path); string trackPath = useRelative ? Path.GetRelativePath(outputPath, tf.Path) : tf.Path; lines.Add($"#EXTINF:-1,{displayName}"); lines.Add(trackPath); } File.WriteAllLines(fullPath, lines, Encoding.UTF8); _logger.Info($"Written {files.Count} track(s) to '{fullPath}'"); } private static string? FindCommonRoot(List paths) { if (paths.Count == 0) return null; if (paths.Count == 1) return Path.GetDirectoryName(paths[0]); List segments = [.. paths.Select(p => Path.GetFullPath(p).Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries))]; int minLen = segments.Min(s => s.Length); List common = []; for (int i = 0; i < minLen; i++) { string seg = segments[0][i]; if (segments.All(s => s[i].Equals(seg, StringComparison.OrdinalIgnoreCase))) common.Add(seg); else break; } if (common.Count == 0) return null; string root = string.Join(Path.DirectorySeparatorChar, common); return common[0].EndsWith(':') ? root + Path.DirectorySeparatorChar : root; } private Dictionary GetSnapshots() => _pluginSettings.GetValue>(SnapshotKey) ?? []; private void SaveSnapshots(Dictionary snapshots) => _pluginSettings.SetValue(SnapshotKey, snapshots); private static string SanitizeFilename(string name) => Path.GetInvalidFileNameChars().Aggregate(name, (s, c) => s.Replace(c, '_')); [GeneratedRegex(@"[^\w]")] private static partial Regex NormalizeRegex(); } ================================================ FILE: Tubifarry/Notifications/PlaylistExport/PlaylistExportSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.Validation; using System.Text.Json; using Tubifarry.Core.Utilities; namespace Tubifarry.Notifications.PlaylistExport; public class PlaylistExportSettingsValidator : AbstractValidator { public PlaylistExportSettingsValidator() => RuleFor(x => x.OutputPath) .NotEmpty() .When(x => !x.AutoDetectOutputPath) .WithMessage("Output path is required when auto-detect is disabled"); } public class PlaylistExportSettings : DynamicStateSettings { private static readonly PlaylistExportSettingsValidator Validator = new(); [FieldDefinition(1, Label = "Output Path", Type = FieldType.Path, HelpText = "Directory where .m3u8 playlist files will be written. Ignored when auto-detect is enabled.")] public string OutputPath { get; set; } = ""; [FieldDefinition(2, Label = "Auto-Detect Output Path", Type = FieldType.Checkbox, HelpText = "Automatically use the common root of your music library as the output directory. Overrides Output Path.")] public bool AutoDetectOutputPath { get; set; } [FieldDefinition(3, Label = "Use Relative Paths", Type = FieldType.Checkbox, HelpText = "Write paths in the .m3u8 files relative to the output directory instead of absolute paths.")] public bool UseRelativePaths { get; set; } [FieldDefinition(4, Label = "Clean Up Playlist on Removal", Type = FieldType.Checkbox, HelpText = "Delete the .m3u8 file when the corresponding import list is removed from Lidarr.")] public bool CleanupOnRemove { get; set; } [FieldDefinition(5, Label = "Track Mode", Type = FieldType.Select, SelectOptions = typeof(PlaylistTrackMode), HelpText = "Controls whether playlists are generated from album-level or track-level data.")] public int TrackMode { get; set; } = (int)PlaylistTrackMode.PreferTrackData; public PlaylistTrackMode GetTrackMode() => (PlaylistTrackMode)TrackMode; public IEnumerable GetSelectedListIds() { Dictionary states = JsonSerializer.Deserialize>( string.IsNullOrEmpty(StateJson) ? "{}" : StateJson) ?? []; return states .Where(kv => kv.Key.StartsWith("list_") && kv.Value) .Select(kv => int.TryParse(kv.Key[5..], out int id) ? id : -1) .Where(id => id > 0); } public override NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum PlaylistTrackMode { [FieldOption(Label = "Album Data Only", Hint = "Always generate a playlist for matched albums.")] AlbumDataOnly = 0, [FieldOption(Label = "Prefer Track Data", Hint = "Use per-track data for lists that support it; fall back to album data for those that don't.")] PreferTrackData = 1, [FieldOption(Label = "Track Data Only", Hint = "Only generate playlists for lists that support per-track data.")] TrackDataOnly = 2, } ================================================ FILE: Tubifarry/Notifications/QueueCleaner/ImportFailureNotificationService.cs ================================================ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Notifications; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Notifications.QueueCleaner { public class ImportFailureNotificationService : IHandle { private readonly INotificationFactory _notificationFactory; private readonly INotificationStatusService _notificationStatusService; private readonly Logger _logger; public ImportFailureNotificationService(INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger) { _notificationFactory = notificationFactory; _notificationStatusService = notificationStatusService; _logger = logger; } private bool ShouldHandleArtist(ProviderDefinition definition, Artist artist) { if (definition.Tags.Empty()) { _logger.Debug("No tags set for this notification."); return true; } if (definition.Tags.Intersect(artist.Tags).Any()) { _logger.Debug("Notification and artist have one or more intersecting tags."); return true; } _logger.Debug("{0} does not have any intersecting tags with {1}. Notification will not be sent.", definition.Name, artist.Name); return false; } public void Handle(AlbumImportIncompleteEvent message) { foreach (INotification? notification in _notificationFactory.OnImportFailureEnabled()) { if (notification is not QueueCleaner queue) continue; try { if (ShouldHandleArtist(notification.Definition, message.TrackedDownload.RemoteAlbum.Artist)) { queue.CleanImport(message); _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send CleanImport: " + notification.Definition.Name); } } } } } ================================================ FILE: Tubifarry/Notifications/QueueCleaner/QueueCleaner.cs ================================================ using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Core.Download; using NzbDrone.Core.Download.TrackedDownloads; using NzbDrone.Core.History; using NzbDrone.Core.Indexers; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Notifications; using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; using System.IO.Abstractions; using Tubifarry.Core.Utilities; using Tubifarry.Notifications.Queue; namespace Tubifarry.Notifications.QueueCleaner { public class QueueCleaner : NotificationBase { private readonly Logger _logger; private readonly IDiskProvider _diskProvider; private readonly ICompletedDownloadService _completedDownloadService; private readonly IHistoryService _historyService; private readonly IEventAggregator _eventAggregator; private readonly INamingConfigService _namingConfig; private readonly IIndexerFactory _indexerFactory; public override string Name => "Queue Cleaner"; public override string Link => ""; public override ProviderMessage Message => new("Queue Cleaner automatically processes items that failed to import. It can rename, blocklist, or remove items based on your settings.", ProviderMessageType.Info); public QueueCleaner(IDiskProvider diskProvider, IHistoryService historyService, INamingConfigService namingConfig, IEventAggregator eventAggregator, IIndexerFactory indexerFactory, ICompletedDownloadService completedDownloadService, Logger logger) { _logger = logger; _indexerFactory = indexerFactory; _diskProvider = diskProvider; _eventAggregator = eventAggregator; _completedDownloadService = completedDownloadService; _historyService = historyService; _namingConfig = namingConfig; } public IFileInfo[] GetAudioFiles(string path, bool allDirectories = true) { if (!_diskProvider.FolderExists(path)) { _logger.Debug("Cannot get audio files: Directory does not exist: {0}", path); return []; } List filesOnDisk = _diskProvider.GetFileInfos(path, allDirectories); return filesOnDisk.Where(file => MediaFileExtensions.Extensions.Contains(file.Extension)).ToArray(); } internal void CleanImport(AlbumImportIncompleteEvent message) { TrackedDownload trackedDownload = message.TrackedDownload; if (!trackedDownload.IsTrackable || trackedDownload.State != TrackedDownloadState.ImportFailed || !trackedDownload.DownloadItem.CanMoveFiles) return; if (Settings.Indexers?.Any() == true && !Settings.Indexers.Contains(trackedDownload.Indexer)) { _logger.Debug($"Skipping queue cleaning for download '{trackedDownload.DownloadItem.Title}'. Indexer '{trackedDownload.Indexer}' is not in list"); return; } if (Settings.SkipReleaseSources != (int)SkipReleaseSourceOptions.Disabled) { ReleaseSourceType releaseSource = trackedDownload.RemoteAlbum?.ReleaseSource ?? ReleaseSourceType.Unknown; if (releaseSource == ReleaseSourceType.Unknown) { List grabbedItems = [.. _historyService.Find(trackedDownload.DownloadItem.DownloadId, EntityHistoryEventType.Grabbed)]; if (grabbedItems.Count > 0) { EntityHistory historyItem = grabbedItems[^1]; _ = Enum.TryParse(historyItem.Data.GetValueOrDefault(EntityHistory.RELEASE_SOURCE, nameof(ReleaseSourceType.Unknown)), out releaseSource); } } _logger.Trace($"Release source for download '{trackedDownload.DownloadItem.Title}': {releaseSource}, Skip threshold: {(SkipReleaseSourceOptions)Settings.SkipReleaseSources}"); if ((int)releaseSource >= Settings.SkipReleaseSources && releaseSource != ReleaseSourceType.Unknown) { _logger.Info($"Skipping queue cleaning for download '{trackedDownload.DownloadItem.Title}'. Release source '{releaseSource}' is at or above skip threshold '{(SkipReleaseSourceOptions)Settings.SkipReleaseSources}'."); return; } } switch (CheckImport(trackedDownload)) { case ImportFailureReason.FailedBecauseOfMissingTracks: HandleFailure(trackedDownload, Settings.ImportCleaningOption, ImportCleaningOptions.WhenMissingTracks); break; case ImportFailureReason.FailedBecauseOfInsufficientInformation: HandleFailure(trackedDownload, Settings.ImportCleaningOption, ImportCleaningOptions.WhenAlbumInfoIncomplete); break; case ImportFailureReason.Both: if (Settings.ImportCleaningOption == (int)ImportCleaningOptions.Disabled) break; HandleFailure(trackedDownload, Settings.ImportCleaningOption, ImportCleaningOptions.Always); break; case ImportFailureReason.DidNotFail: break; } } private void HandleFailure(TrackedDownload trackedDownload, int importCleaningOption, ImportCleaningOptions requiredOption) { if (importCleaningOption != (int)requiredOption && importCleaningOption != (int)ImportCleaningOptions.Always) return; if (Settings.RenameOption != (int)RenameOptions.DoNotRename && Rename(trackedDownload)) { Retry(trackedDownload); return; } if (Settings.BlocklistOption == (int)BlocklistOptions.RemoveAndBlocklist || Settings.BlocklistOption == (int)BlocklistOptions.BlocklistOnly) Blocklist(trackedDownload); if (Settings.BlocklistOption == (int)BlocklistOptions.RemoveAndBlocklist || Settings.BlocklistOption == (int)BlocklistOptions.RemoveOnly) Remove(trackedDownload); } private void Remove(TrackedDownload item) { if (!item.DownloadItem.CanBeRemoved) return; string downloadPath = item.DownloadItem.OutputPath.FullPath; if (!_diskProvider.FolderExists(downloadPath)) { _logger.Debug("Skipping file cleanup: Download directory no longer exists: {0}", downloadPath); _eventAggregator.PublishEvent(new DownloadCanBeRemovedEvent(item)); return; } foreach (IFileInfo file in (List)_diskProvider.GetFileInfos(downloadPath, true)) _diskProvider.DeleteFile(file.FullName); _eventAggregator.PublishEvent(new DownloadCanBeRemovedEvent(item)); } private bool Rename(TrackedDownload item) { if (!item.DownloadItem.CanMoveFiles) { _logger.Debug("Skipping rename: Download item cannot be moved."); return false; } string downloadPath = item.DownloadItem.OutputPath.FullPath; if (!_diskProvider.FolderExists(downloadPath)) { _logger.Debug("Skipping rename: Download directory no longer exists: {0}", downloadPath); return false; } List filesOnDisk = [.. _diskProvider.GetFileInfos(downloadPath, true)]; HashSet audioExtensions = new(MediaFileExtensions.Extensions, StringComparer.OrdinalIgnoreCase); ReleaseFormatter releaseFormatter = new(item.RemoteAlbum.Release, item.RemoteAlbum.Artist, _namingConfig.GetConfig()); return filesOnDisk.Any(file => TryRenameFile(file, audioExtensions, releaseFormatter, filesOnDisk)); } private bool TryRenameFile(IFileInfo file, HashSet audioExtensions, ReleaseFormatter releaseFormatter, List filesOnDisk) { string filePath = file.FullName; string? directory = Path.GetDirectoryName(filePath); if (directory == null || !audioExtensions.Contains(Path.GetExtension(filePath))) { _logger.Warn($"Unable to determine directory for file: {filePath}"); return false; } try { TagLib.File fileTags = TagLib.File.Create(filePath); string artistName = fileTags.Tag.FirstAlbumArtist ?? fileTags.Tag.FirstPerformer ?? "UnknownArtist"; string title = fileTags.Tag.Title ?? Path.GetFileNameWithoutExtension(filePath); string trackNumber = fileTags.Tag.Track.ToString("D2"); string newFileNameWithoutExtension = releaseFormatter.BuildTrackFilename(null, new Track { Title = title, TrackNumber = trackNumber, Artist = new Artist { Name = artistName } }, new Album { Title = title }); string newFileName = $"{newFileNameWithoutExtension}{Path.GetExtension(filePath)}"; string newFilePath = Path.Combine(directory, newFileName); if (!newFilePath.Equals(filePath, StringComparison.OrdinalIgnoreCase) && MoveFile(filePath, newFilePath)) { string oldBaseName = Path.GetFileNameWithoutExtension(filePath); filesOnDisk.Where(f => !Path.GetExtension(f.FullName).Equals(Path.GetExtension(filePath), StringComparison.OrdinalIgnoreCase) && Path.GetFileNameWithoutExtension(f.FullName).Equals(oldBaseName, StringComparison.OrdinalIgnoreCase)).ToList() .ForEach(associatedFile => MoveFile(associatedFile.FullName, Path.Combine(directory, $"{newFileNameWithoutExtension}{Path.GetExtension(associatedFile.FullName)}"))); return true; } } catch (Exception ex) { _logger.Error(ex, $"Failed to parse or rename file: {filePath}"); } return false; } private bool MoveFile(string sourcePath, string destinationPath) { if (!_diskProvider.FileExists(sourcePath)) return false; try { _diskProvider.MoveFile(sourcePath, destinationPath); return true; } catch (Exception ex) { _logger.Error(ex, $"Failed to rename file: {sourcePath}"); return false; } } private void Blocklist(TrackedDownload item) { item.State = TrackedDownloadState.DownloadFailed; List grabbedItems = [.. _historyService.Find(item.DownloadItem.DownloadId, EntityHistoryEventType.Grabbed)]; EntityHistory historyItem = grabbedItems[^1]; _ = Enum.TryParse(historyItem.Data.GetValueOrDefault(EntityHistory.RELEASE_SOURCE, nameof(ReleaseSourceType.Unknown)), out ReleaseSourceType releaseSource); DownloadFailedEvent downloadFailedEvent = new() { ArtistId = historyItem.ArtistId, AlbumIds = grabbedItems.Select(h => h.AlbumId).Distinct().ToList(), Quality = historyItem.Quality, SourceTitle = historyItem.SourceTitle, DownloadClient = historyItem.Data.GetValueOrDefault(EntityHistory.DOWNLOAD_CLIENT), DownloadId = historyItem.DownloadId, Message = "Import failed: Item removed by Queue Cleaner.", Data = historyItem.Data, TrackedDownload = item, SkipRedownload = !Settings.RetryFindingRelease, ReleaseSource = releaseSource }; _eventAggregator.PublishEvent(downloadFailedEvent); } private void Retry(TrackedDownload item) { item.State = TrackedDownloadState.ImportPending; _completedDownloadService.Import(item); } private static ImportFailureReason CheckImport(TrackedDownload trackedDownload) { if (trackedDownload.State != TrackedDownloadState.ImportFailed) return ImportFailureReason.DidNotFail; bool hasMissingTracks = trackedDownload.StatusMessages .Any(sm => sm.Messages.Any(m => m.Contains("Has missing tracks", StringComparison.OrdinalIgnoreCase))); bool hasInsufficientInformation = trackedDownload.StatusMessages .Any(sm => sm.Messages.Any(m => m.Contains("Album match is not close enough", StringComparison.OrdinalIgnoreCase))); return (hasMissingTracks, hasInsufficientInformation) switch { (true, true) => ImportFailureReason.Both, (true, _) => ImportFailureReason.FailedBecauseOfMissingTracks, (_, true) => ImportFailureReason.FailedBecauseOfInsufficientInformation, _ => ImportFailureReason.DidNotFail }; } public override void OnImportFailure(AlbumDownloadMessage message) => base.OnImportFailure(message); public override ValidationResult Test() { ValidationResult result = new(); IEnumerable validProviders = _indexerFactory.GetAvailableProviders().Select(x => x.Name); if (Settings.Indexers?.Any() == true) { List invalidProviders = Settings.Indexers .Where(indexer => !validProviders.Contains(indexer, StringComparer.OrdinalIgnoreCase)) .ToList(); if (invalidProviders.Count != 0) { string errorMessage = $"The following indexers are not valid or available: {string.Join(", ", invalidProviders)}"; result.Errors.Add(new ValidationFailure(nameof(Settings.Indexers), errorMessage, Settings.Indexers)); } } return result; } public enum ImportFailureReason { DidNotFail, Both, FailedBecauseOfMissingTracks, FailedBecauseOfInsufficientInformation } } } ================================================ FILE: Tubifarry/Notifications/QueueCleaner/QueueCleanerSettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Notifications.Queue { public class QueueCleanerSettingsValidator : AbstractValidator; public class QueueCleanerSettings : IProviderConfig { private static readonly QueueCleanerSettingsValidator Validator = new(); [FieldDefinition(1, Label = "Blocklist Option", Type = FieldType.Select, SelectOptions = typeof(BlocklistOptions), HelpText = "Specify how to handle blocklisting during queue cleaning.")] public int BlocklistOption { get; set; } = (int)BlocklistOptions.RemoveAndBlocklist; [FieldDefinition(2, Label = "Rename Option", Type = FieldType.Select, SelectOptions = typeof(RenameOptions), HelpText = "Specify how to handle renaming during queue cleaning.", HelpTextWarning = "Only enable this option if you plan to rename your songs again later (e.g., during Lidarr import), as it may disrupt the current naming structure.")] public int RenameOption { get; set; } = (int)RenameOptions.DoNotRename; [FieldDefinition(3, Label = "Import Cleaning Option", Type = FieldType.Select, SelectOptions = typeof(ImportCleaningOptions), HelpText = "Specify how to handle import cleaning during queue cleaning.")] public int ImportCleaningOption { get; set; } = (int)ImportCleaningOptions.Always; [FieldDefinition(4, Label = "Skip Release Sources", Type = FieldType.Select, SelectOptions = typeof(SkipReleaseSourceOptions), HelpText = "Skip processing downloads from these release sources and below, allowing manual import control.")] public int SkipReleaseSources { get; set; } = (int)SkipReleaseSourceOptions.Disabled; [FieldDefinition(5, Label = "Retry Finding Release", Type = FieldType.Checkbox, HelpText = "Retry searching for the release if the import fails during queue cleaning.")] public bool RetryFindingRelease { get; set; } = true; [FieldDefinition(6, Label = "Indexers", Type = FieldType.Tag, HelpText = "Names of indexers to watch. Leave empty to use all available indexers.")] public IEnumerable Indexers { get; set; } = []; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } public enum BlocklistOptions { [FieldOption(Label = "Remove and Blocklist", Hint = "Remove the album and add it to the blocklist to prevent future imports.")] RemoveAndBlocklist, [FieldOption(Label = "Remove Only", Hint = "Remove the album without adding it to the blocklist.")] RemoveOnly, [FieldOption(Label = "Blocklist Only", Hint = "Add the album to the blocklist without removing it.")] BlocklistOnly } public enum RenameOptions { [FieldOption(Label = "Do Not Rename", Hint = "No renaming will be performed on the album folder or tracks during import.")] DoNotRename, [FieldOption(Label = "Rename Tracks", Hint = "Rename the album tracks based on available metadata, then retry the import process.")] RenameTracks } public enum ImportCleaningOptions { [FieldOption(Label = "Disabled", Hint = "No cleaning or organization will be performed during import.")] Disabled, [FieldOption(Label = "When Missing Tracks", Hint = "Clean the album if it has missing tracks.")] WhenMissingTracks, [FieldOption(Label = "When Album Info Incomplete", Hint = "Clean the album if the metadata is incomplete or insufficient.")] WhenAlbumInfoIncomplete, [FieldOption(Label = "Always", Hint = "Clean the album, regardless of metadata or track completeness.")] Always } public enum SkipReleaseSourceOptions { [FieldOption(Label = "Disabled", Hint = "Process all downloads regardless of source.")] Disabled = 0, [FieldOption(Label = "Automated", Hint = "Skip RSS and all search types.")] Rss = 1, [FieldOption(Label = "Search", Hint = "Skip search downloads (Search, User Search, Interactive).")] Search = 2, [FieldOption(Label = "User Invoked Search", Hint = "Skip user-invoked searches (User Search, Interactive).")] UserInvokedSearch = 3, [FieldOption(Label = "Interactive Search", Hint = "Skip interactive search downloads.")] InteractiveSearch = 4 } } ================================================ FILE: Tubifarry/Notifications/YouTubeProxy/YouTubeProxyNotification.cs ================================================ using FluentValidation.Results; using NzbDrone.Core.Notifications; using NzbDrone.Core.ThingiProvider; namespace Tubifarry.Notifications.YouTubeProxy { public class YouTubeProxyNotification : NotificationBase { public override string Name => "YouTube Proxy"; public override string Link => ""; public override ProviderMessage Message => new("YouTube Proxy configures SOCKS5 proxy settings for HTTP requests. Configure your shadowsocks proxy details to route traffic through the specified server.", ProviderMessageType.Info); public override ValidationResult Test() => new(); public override void OnGrab(GrabMessage message) { } } } ================================================ FILE: Tubifarry/Notifications/YouTubeProxy/YouTubeProxyService.cs ================================================ using DownloadAssistant.Base; using NLog; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Notifications; using System.Net; using System.Net.Security; using System.Security.Authentication; namespace Tubifarry.Notifications.YouTubeProxy { public class YouTubeProxyService : IHandle { private readonly INotificationFactory _notificationFactory; private readonly INotificationStatusService _notificationStatusService; private readonly Logger _logger; public YouTubeProxyService(INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger) { _notificationFactory = notificationFactory; _notificationStatusService = notificationStatusService; _logger = logger; } public void Handle(ApplicationStartedEvent message) { try { ConfigureProxySettings(); } catch (Exception ex) { _logger.Error(ex, "Failed to configure proxy settings on application startup"); } } private void ConfigureProxySettings() { foreach (INotification? notification in (List)_notificationFactory.GetAvailableProviders()) { if (notification is not YouTubeProxyNotification proxyNotification) continue; if (notification.Definition is not NotificationDefinition definition || !definition.Enable) { _logger.Debug("YouTubeProxy notification is disabled, skipping proxy configuration"); continue; } YouTubeProxySettings settings = (YouTubeProxySettings)proxyNotification.Definition.Settings; try { ConfigureHttpClientWithProxy(settings); _notificationStatusService.RecordSuccess(notification.Definition.Id); _logger.Info("Successfully configured HTTP client with SOCKS5 proxy: {0}:{1}", settings.ProxyHost, settings.ProxyPort); } catch (Exception ex) { _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Error(ex, "Failed to configure HTTP client with proxy settings"); } } } private void ConfigureHttpClientWithProxy(YouTubeProxySettings settings) { _logger.Info("Configuring HTTP client with SOCKS5 proxy: {0}:{1}", settings.ProxyHost, settings.ProxyPort); WebProxy proxy; if (settings.RequireAuthentication && !string.IsNullOrWhiteSpace(settings.Username)) { proxy = new WebProxy($"socks5://{settings.ProxyHost}:{settings.ProxyPort}") { Credentials = new NetworkCredential(settings.Username, settings.Password) }; _logger.Debug("Proxy configured with authentication for user: {0}", settings.Username); } else { proxy = new WebProxy($"socks5://{settings.ProxyHost}:{settings.ProxyPort}"); _logger.Debug("Proxy configured without authentication"); } SocketsHttpHandler handler = new() { Proxy = proxy, UseProxy = true, PooledConnectionLifetime = TimeSpan.FromMinutes(10), PooledConnectionIdleTimeout = TimeSpan.FromMinutes(5), MaxConnectionsPerServer = 10, SslOptions = new SslClientAuthenticationOptions { EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13 } }; HttpClient client = new(handler, disposeHandler: true) { Timeout = TimeSpan.FromSeconds(100) }; try { client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentBuilder.Generate()); } catch (Exception ex) { _logger.Warn(ex, "Failed to set user agent header, continuing without it"); } HttpGet.HttpClient = client; } } } ================================================ FILE: Tubifarry/Notifications/YouTubeProxy/YouTubeProxySettings.cs ================================================ using FluentValidation; using NzbDrone.Core.Annotations; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace Tubifarry.Notifications.YouTubeProxy { public class YouTubeProxySettingsValidator : AbstractValidator { public YouTubeProxySettingsValidator() { RuleFor(c => c.ProxyHost) .NotEmpty() .WithMessage("Proxy host is required when proxy is enabled"); RuleFor(c => c.ProxyPort) .GreaterThan(0) .LessThanOrEqualTo(65535) .WithMessage("Proxy port must be between 1 and 65535"); RuleFor(c => c.Username) .NotEmpty() .When(c => c.RequireAuthentication) .WithMessage("Username is required when authentication is enabled"); RuleFor(c => c.Password) .NotEmpty() .When(c => c.RequireAuthentication) .WithMessage("Password is required when authentication is enabled"); } } public class YouTubeProxySettings : IProviderConfig { private static readonly YouTubeProxySettingsValidator Validator = new(); [FieldDefinition(1, Label = "Proxy Host", Type = FieldType.Textbox, HelpText = "The hostname or IP address of your SOCKS5 proxy server.")] public string ProxyHost { get; set; } = ""; [FieldDefinition(2, Label = "Proxy Port", Type = FieldType.Number, HelpText = "The port number of your SOCKS5 proxy server (typically 1080).")] public int ProxyPort { get; set; } = 1080; [FieldDefinition(3, Label = "Require Authentication", Type = FieldType.Checkbox, HelpText = "Enable if your SOCKS5 proxy requires username and password authentication.")] public bool RequireAuthentication { get; set; } [FieldDefinition(4, Label = "Username", Type = FieldType.Textbox, HelpText = "Username for proxy authentication (only required if authentication is enabled).")] public string Username { get; set; } = ""; [FieldDefinition(5, Label = "Password", Type = FieldType.Password, HelpText = "Password for proxy authentication (only required if authentication is enabled).")] public string Password { get; set; } = ""; public NzbDroneValidationResult Validate() => new(Validator.Validate(this)); } } ================================================ FILE: Tubifarry/Plugin.cs ================================================ using NLog; using NLog.Config; using NzbDrone.Core.Indexers; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Plugins; #if CI using NzbDrone.Core.Plugins.Commands; #endif using NzbDrone.Core.Profiles.Delay; using Tubifarry.Core.Utilities; #if !MASTER_BRANCH using Tubifarry.Core.Telemetry; #endif namespace Tubifarry { public class Tubifarry : Plugin #if !MASTER_BRANCH , IHandle , IHandle #endif { private readonly Logger _logger; private readonly Lazy _pluginService; private readonly IManageCommandQueue _commandQueueManager; private readonly IPluginSettings _pluginSettings; public const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; //$"{PluginInfo.Name}/{PluginInfo.AssemblyVersion} ({PluginInfo.Framework} {PluginInfo.Branch})"; public override string Name => PluginInfo.Name; public override string Owner => PluginInfo.Author; public override string GithubUrl => PluginInfo.RepoUrl; private static Type[] ProtocolTypes => [typeof(YoutubeDownloadProtocol), typeof(SoulseekDownloadProtocol), typeof(LucidaDownloadProtocol), typeof(QobuzDownloadProtocol), typeof(SubSonicDownloadProtocol), typeof(AmazonMusicDownloadProtocol)]; public static TimeSpan AverageRuntime { get; private set; } = TimeSpan.FromDays(4); public static DateTime LastStarted { get; private set; } = DateTime.UtcNow; public Tubifarry(IDelayProfileRepository repo, IPluginSettings pluginSettings, IEnumerable downloadProtocols, Lazy pluginService, IManageCommandQueue commandQueueManager, Logger logger) { _logger = logger; _commandQueueManager = commandQueueManager; _pluginService = pluginService; _pluginSettings = pluginSettings; CheckDelayProfiles(repo, downloadProtocols); } private void CheckDelayProfiles(IDelayProfileRepository repo, IEnumerable downloadProtocols) { foreach (IDownloadProtocol protocol in downloadProtocols.Where(x => ProtocolTypes.Any(y => y == x.GetType()))) { _logger.Trace($"Checking protocol: {protocol.GetType().Name}"); foreach (DelayProfile? profile in repo.All()) { if (!profile.Items.Any(x => x.Protocol == protocol.GetType().Name)) { _logger.Debug($"Added protocol to DelayProfile (ID: {profile.Id})"); profile.Items.Add(GetProtocolItem(protocol, true)); repo.Update(profile); } } } } private static DelayProfileProtocolItem GetProtocolItem(IDownloadProtocol protocol, bool allowed) => new() { Name = protocol.GetType().Name.Replace("DownloadProtocol", ""), Protocol = protocol.GetType().Name, Allowed = allowed }; public void Handle(ApplicationStartingEvent message) { #if !MASTER_BRANCH TubifarrySentry.Initialize(); if (TubifarrySentry.IsEnabled) { TubifarrySentryTarget target = new TubifarrySentryTarget { Name = "tubifarry", Layout = "${message}", Enabled = true, MinimumBreadcrumbLevel = LogLevel.Debug, MinimumEventLevel = LogLevel.Error }; LoggingRule rule = new LoggingRule("Tubifarry*", LogLevel.Warn, target); LogManager.Configuration.AddTarget(target); LogManager.Configuration.LoggingRules.Add(rule); LogManager.ReconfigExistingLoggers(); } #endif #if CI AvailableVersion = _pluginService.Value.GetRemotePlugin(GithubUrl).Version; if (AvailableVersion > InstalledVersion) _commandQueueManager.Push(new InstallPluginCommand() { GithubUrl = GithubUrl }); #endif List lastStarted = _pluginSettings.GetValue>("lastStarted") ?? []; LastStarted = DateTime.UtcNow; lastStarted.Add(LastStarted); if (lastStarted.Count > 10) lastStarted.RemoveAt(0); _pluginSettings.SetValue("lastStarted", lastStarted); if (lastStarted.Count > 1) { lastStarted.Sort(); TimeSpan totalRuntime = TimeSpan.Zero; for (int i = 1; i < lastStarted.Count; i++) { TimeSpan timeBetweenStarts = lastStarted[i] - lastStarted[i - 1]; if (timeBetweenStarts < TimeSpan.FromDays(30)) totalRuntime += timeBetweenStarts; } int validIntervals = Math.Max(1, lastStarted.Count - 1); AverageRuntime = TimeSpan.FromTicks(totalRuntime.Ticks / validIntervals); _logger.Debug($"Average runtime between restarts is {AverageRuntime.TotalDays:F2} days"); } } #if !MASTER_BRANCH public void Handle(ApplicationShutdownRequested message) { TubifarrySentry.Shutdown(); } #endif } } ================================================ FILE: Tubifarry/PluginInfo.targets ================================================ $(IntermediateOutputPath)PluginInfo.cs ================================================ FILE: Tubifarry/PluginKeys.targets ================================================  $(IntermediateOutputPath)PluginKeys.cs ================================================ FILE: Tubifarry/PreBuild.targets ================================================ $(MSBuildProjectDirectory)\..\Submodules\Lidarr $(SubmoduleDir)\src\NzbDrone.Core\Lidarr.Core.csproj ================================================ FILE: Tubifarry/Tubifarry.csproj ================================================  net8.0 embedded enable enable Lidarr.Plugin.$(MSBuildProjectName) 1.0.0.0 true TypNull https://github.com/TypNull/Tubifarry false $(AssemblyVersion)-$(Branch) $(DefineConstants);CI $(DefineConstants);MASTER_BRANCH $(DefineConstants);DEV_BRANCH $(DefineConstants);FEATURE_BRANCH all runtime; build; native; contentfiles; analyzers; buildtransitive false runtime all runtime; build; native; contentfiles; analyzers ================================================ FILE: Tubifarry.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 18 VisualStudioVersion = 18.1.11312.151 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tubifarry", "Tubifarry\Tubifarry.csproj", "{003DFC9A-18D9-47DB-BFC3-5375AC29B469}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Core", "Submodules\Lidarr\src\NzbDrone.Core\Lidarr.Core.csproj", "{91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Common", "Submodules\Lidarr\src\NzbDrone.Common\Lidarr.Common.csproj", "{E1E12DF8-D168-4259-96C9-B7A6EE58A4AF}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Submodules", "Submodules", "{F8B21985-D2D8-4556-B436-E3B3E9859062}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Host", "Submodules\Lidarr\src\NzbDrone.Host\Lidarr.Host.csproj", "{97AAE5CB-DE41-35CC-994B-1E52C8FDFA60}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Api.V1", "Submodules\Lidarr\src\Lidarr.Api.V1\Lidarr.Api.V1.csproj", "{2043F410-D541-0B5C-1CC8-0133DB1D8B39}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Http", "Submodules\Lidarr\src\Lidarr.Http\Lidarr.Http.csproj", "{E3AEFFDA-ABE8-A47E-202E-BC445822FB89}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.SignalR", "Submodules\Lidarr\src\NzbDrone.SignalR\Lidarr.SignalR.csproj", "{5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Console", "Submodules\Lidarr\src\NzbDrone.Console\Lidarr.Console.csproj", "{15129560-39D3-E118-4056-49656F5A637C}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Lidarr", "Lidarr", "{11025488-4B11-4269-966E-4C2B04D1B5C3}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Windows", "Submodules\Lidarr\src\NzbDrone.Windows\Lidarr.Windows.csproj", "{B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lidarr.Mono", "Submodules\Lidarr\src\NzbDrone.Mono\Lidarr.Mono.csproj", "{0AC5781C-A310-AF9D-D097-4FF1F0FF8A68}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {003DFC9A-18D9-47DB-BFC3-5375AC29B469}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {003DFC9A-18D9-47DB-BFC3-5375AC29B469}.Debug|Any CPU.Build.0 = Debug|Any CPU {003DFC9A-18D9-47DB-BFC3-5375AC29B469}.Release|Any CPU.ActiveCfg = Release|Any CPU {003DFC9A-18D9-47DB-BFC3-5375AC29B469}.Release|Any CPU.Build.0 = Release|Any CPU {91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A}.Debug|Any CPU.Build.0 = Debug|Any CPU {91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A}.Release|Any CPU.ActiveCfg = Release|Any CPU {91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A}.Release|Any CPU.Build.0 = Release|Any CPU {E1E12DF8-D168-4259-96C9-B7A6EE58A4AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E1E12DF8-D168-4259-96C9-B7A6EE58A4AF}.Debug|Any CPU.Build.0 = Debug|Any CPU {E1E12DF8-D168-4259-96C9-B7A6EE58A4AF}.Release|Any CPU.ActiveCfg = Release|Any CPU {E1E12DF8-D168-4259-96C9-B7A6EE58A4AF}.Release|Any CPU.Build.0 = Release|Any CPU {97AAE5CB-DE41-35CC-994B-1E52C8FDFA60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {97AAE5CB-DE41-35CC-994B-1E52C8FDFA60}.Debug|Any CPU.Build.0 = Debug|Any CPU {97AAE5CB-DE41-35CC-994B-1E52C8FDFA60}.Release|Any CPU.ActiveCfg = Release|Any CPU {97AAE5CB-DE41-35CC-994B-1E52C8FDFA60}.Release|Any CPU.Build.0 = Release|Any CPU {2043F410-D541-0B5C-1CC8-0133DB1D8B39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2043F410-D541-0B5C-1CC8-0133DB1D8B39}.Debug|Any CPU.Build.0 = Debug|Any CPU {2043F410-D541-0B5C-1CC8-0133DB1D8B39}.Release|Any CPU.ActiveCfg = Release|Any CPU {2043F410-D541-0B5C-1CC8-0133DB1D8B39}.Release|Any CPU.Build.0 = Release|Any CPU {E3AEFFDA-ABE8-A47E-202E-BC445822FB89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E3AEFFDA-ABE8-A47E-202E-BC445822FB89}.Debug|Any CPU.Build.0 = Debug|Any CPU {E3AEFFDA-ABE8-A47E-202E-BC445822FB89}.Release|Any CPU.ActiveCfg = Release|Any CPU {E3AEFFDA-ABE8-A47E-202E-BC445822FB89}.Release|Any CPU.Build.0 = Release|Any CPU {5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09}.Debug|Any CPU.Build.0 = Debug|Any CPU {5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09}.Release|Any CPU.ActiveCfg = Release|Any CPU {5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09}.Release|Any CPU.Build.0 = Release|Any CPU {15129560-39D3-E118-4056-49656F5A637C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {15129560-39D3-E118-4056-49656F5A637C}.Debug|Any CPU.Build.0 = Debug|Any CPU {15129560-39D3-E118-4056-49656F5A637C}.Release|Any CPU.ActiveCfg = Release|Any CPU {15129560-39D3-E118-4056-49656F5A637C}.Release|Any CPU.Build.0 = Release|Any CPU {B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8}.Debug|Any CPU.Build.0 = Debug|Any CPU {B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8}.Release|Any CPU.ActiveCfg = Release|Any CPU {B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8}.Release|Any CPU.Build.0 = Release|Any CPU {0AC5781C-A310-AF9D-D097-4FF1F0FF8A68}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0AC5781C-A310-AF9D-D097-4FF1F0FF8A68}.Debug|Any CPU.Build.0 = Debug|Any CPU {0AC5781C-A310-AF9D-D097-4FF1F0FF8A68}.Release|Any CPU.ActiveCfg = Release|Any CPU {0AC5781C-A310-AF9D-D097-4FF1F0FF8A68}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {91CB11D7-4E7A-4B1A-AE6A-15E1F075C27A} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {E1E12DF8-D168-4259-96C9-B7A6EE58A4AF} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {97AAE5CB-DE41-35CC-994B-1E52C8FDFA60} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {2043F410-D541-0B5C-1CC8-0133DB1D8B39} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {E3AEFFDA-ABE8-A47E-202E-BC445822FB89} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {5F3AB49B-9442-4A4C-246E-FC3FAE8E8E09} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {15129560-39D3-E118-4056-49656F5A637C} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {11025488-4B11-4269-966E-4C2B04D1B5C3} = {F8B21985-D2D8-4556-B436-E3B3E9859062} {B5D972C6-9EA6-6AEF-E20C-B8E85E05DDE8} = {11025488-4B11-4269-966E-4C2B04D1B5C3} {0AC5781C-A310-AF9D-D097-4FF1F0FF8A68} = {11025488-4B11-4269-966E-4C2B04D1B5C3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FE0E5C13-34E6-4D96-8912-D2574B06FA0D} EndGlobalSection EndGlobal ================================================ FILE: stylecop.json ================================================ { "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", "settings": { "documentationRules": { "xmlHeader": false, "documentInterfaces": false, "documentInternalElements": false }, "indentation": { "indentationSize": 4, "useTabs": false }, "layoutRules": { "newlineAtEndOfFile": "require" }, "orderingRules": { "systemUsingDirectivesFirst": true, "usingDirectivesPlacement": "outsideNamespace" } } }