Showing preview only (7,946K chars total). Download the full file or copy to clipboard to get everything.
Repository: GetStream/Winds
Branch: master
Commit: 23c0e6d44cf5
Files: 445
Total size: 7.5 MB
Directory structure:
gitextract_ss1x8z9z/
├── .eslintrc.js
├── .gitattributes
├── .github/
│ ├── ISSUE_TEMPLATE.md
│ ├── PULL_REQUEST_TEMPLATE.md
│ └── stale.yml
├── .gitignore
├── .prettierrc
├── .travis.yml
├── Dockerfile
├── LICENSE
├── README.md
├── RSS.md
├── STYLE.md
├── api/
│ ├── Dockerfile
│ ├── build.sh
│ ├── config.yaml
│ ├── docker-compose.yml
│ ├── ecosystem.dev.config.js
│ ├── now.json
│ ├── package.json
│ ├── scripts/
│ │ ├── docker-build.sh
│ │ ├── docker-compose-aws.sh
│ │ ├── docker-compose.sh
│ │ └── make-build.sh
│ ├── setup-tests.js
│ ├── src/
│ │ ├── .babelrc
│ │ ├── .prettierignore
│ │ ├── asyncTasks.js
│ │ ├── commands/
│ │ │ ├── _debug-feed.js
│ │ │ ├── cleanup-follows.js
│ │ │ ├── denormalize-follows.js
│ │ │ ├── denormalize-pins.js
│ │ │ ├── email.js
│ │ │ ├── load-featured-feeds.js
│ │ │ ├── rescrape-favicon.js
│ │ │ ├── rescrape-og.js
│ │ │ ├── reset-parsing-state.js
│ │ │ ├── resync-follows.js
│ │ │ ├── winds-article.js
│ │ │ ├── winds-discover.js
│ │ │ ├── winds-merge.js
│ │ │ ├── winds-og.js
│ │ │ ├── winds-podcast.js
│ │ │ ├── winds-rebuild-search.js
│ │ │ ├── winds-rehash.js
│ │ │ ├── winds-rss.js
│ │ │ ├── winds-truncate-rss-feed.js
│ │ │ └── winds.js
│ │ ├── config/
│ │ │ ├── dev.js
│ │ │ ├── index.js
│ │ │ ├── prod.js
│ │ │ └── test.js
│ │ ├── controllers/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── default.js
│ │ │ ├── email.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── feed.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── fixtures/
│ │ │ └── featured.json
│ │ ├── loadenv.js
│ │ ├── models/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── content.js
│ │ │ ├── enclosure.js
│ │ │ ├── episode.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── parsers/
│ │ │ ├── content.js
│ │ │ ├── detect-language.js
│ │ │ ├── detect-type.js
│ │ │ ├── discovery.js
│ │ │ ├── feed.js
│ │ │ └── og.js
│ │ ├── routes/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── email.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── index.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── server.js
│ │ ├── utils/
│ │ │ ├── analytics.js
│ │ │ ├── basicAuth.js
│ │ │ ├── blockedURLs.js
│ │ │ ├── collections.js
│ │ │ ├── controllers.js
│ │ │ ├── db/
│ │ │ │ └── index.js
│ │ │ ├── email/
│ │ │ │ ├── context.js
│ │ │ │ ├── send.js
│ │ │ │ └── templates/
│ │ │ │ ├── daily.ejs
│ │ │ │ ├── followee.ejs
│ │ │ │ ├── password.ejs
│ │ │ │ ├── weekly.ejs
│ │ │ │ └── welcome.ejs
│ │ │ ├── errors.js
│ │ │ ├── logger/
│ │ │ │ ├── index.js
│ │ │ │ └── sentry.js
│ │ │ ├── merge.js
│ │ │ ├── personalization/
│ │ │ │ └── index.js
│ │ │ ├── queue.js
│ │ │ ├── random.js
│ │ │ ├── rate-limiter.js
│ │ │ ├── sanitize.js
│ │ │ ├── search/
│ │ │ │ └── index.js
│ │ │ ├── social.js
│ │ │ ├── statsd.js
│ │ │ ├── stream.js
│ │ │ ├── upsert.js
│ │ │ ├── urls.js
│ │ │ ├── validation.js
│ │ │ └── watchdog.js
│ │ └── workers/
│ │ ├── conductor.js
│ │ ├── og.js
│ │ ├── podcast.js
│ │ ├── rss.js
│ │ ├── social.js
│ │ ├── stream.js
│ │ ├── winds-hackernews.js
│ │ └── winds-queue-state-monitor.js
│ ├── test/
│ │ ├── controllers/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── feed.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── data/
│ │ │ ├── 404.opml
│ │ │ ├── discovery/
│ │ │ │ ├── case.html
│ │ │ │ ├── fail.xml
│ │ │ │ ├── index.html
│ │ │ │ ├── nofavicon.html
│ │ │ │ ├── nourl.xml
│ │ │ │ └── rss.xml
│ │ │ ├── feed/
│ │ │ │ ├── 90.cx
│ │ │ │ ├── a16z
│ │ │ │ ├── api.prprpr.me
│ │ │ │ ├── apublica.org
│ │ │ │ ├── audiworld.com
│ │ │ │ ├── boingboing
│ │ │ │ ├── bookshadow.com
│ │ │ │ ├── dingxiaoyun555.blog.163.com
│ │ │ │ ├── django
│ │ │ │ ├── douban.com
│ │ │ │ ├── empty
│ │ │ │ ├── geektopia.es
│ │ │ │ ├── habr
│ │ │ │ ├── hackernews
│ │ │ │ ├── hackernoon-daily-dev
│ │ │ │ ├── kaiak.tw
│ │ │ │ ├── kottke
│ │ │ │ ├── lemonde
│ │ │ │ ├── lobsters
│ │ │ │ ├── lowendbox.com
│ │ │ │ ├── malformed-hackernews
│ │ │ │ ├── maxwell-land-surveying.com
│ │ │ │ ├── medium-technology
│ │ │ │ ├── perezhilton
│ │ │ │ ├── reddit-r-programming
│ │ │ │ ├── rss.cnki.net
│ │ │ │ ├── ruby-on-rails
│ │ │ │ ├── seattle.craigslist.org
│ │ │ │ ├── shanzhuoboshi.com
│ │ │ │ ├── sospc.name
│ │ │ │ ├── straitstimes.com
│ │ │ │ ├── strava
│ │ │ │ ├── stream
│ │ │ │ ├── techcrunch
│ │ │ │ ├── tejiendoelmundo.wordpress.com
│ │ │ │ ├── thewildeternal.com
│ │ │ │ ├── tmz
│ │ │ │ ├── torrentedigital.com
│ │ │ │ ├── totoyao.wordpress.com
│ │ │ │ ├── treehugger-latest
│ │ │ │ ├── ttt.tt
│ │ │ │ ├── xda-developers.com
│ │ │ │ └── zhukun.net
│ │ │ ├── not-a-url.opml
│ │ │ ├── og/
│ │ │ │ ├── bildblog.html
│ │ │ │ ├── google.html
│ │ │ │ ├── kotaku.html
│ │ │ │ ├── techcrunch.html
│ │ │ │ ├── techcrunch_broken.html
│ │ │ │ └── techcrunch_instagram.html
│ │ │ ├── podcast-feed/
│ │ │ │ ├── a16z
│ │ │ │ ├── atlantamonster
│ │ │ │ ├── buffering-the-vampire-slayer
│ │ │ │ ├── design-details
│ │ │ │ ├── giant-bombcast
│ │ │ │ ├── making-obama
│ │ │ │ ├── nancy
│ │ │ │ ├── serial
│ │ │ │ ├── still-processing
│ │ │ │ └── thehabitat
│ │ │ └── test.xml
│ │ ├── fixtures/
│ │ │ ├── aliases.json
│ │ │ ├── articles.json
│ │ │ ├── featured.json
│ │ │ ├── folders.json
│ │ │ ├── follows.json
│ │ │ ├── initial-data.json
│ │ │ ├── liked-shares.json
│ │ │ ├── listens.json
│ │ │ ├── merge-data.json
│ │ │ ├── notes.json
│ │ │ ├── opml.json
│ │ │ ├── pins.json
│ │ │ ├── playlists.json
│ │ │ ├── tags.json
│ │ │ ├── unstable-guid.json
│ │ │ ├── user.json
│ │ │ └── user_model.json
│ │ ├── models/
│ │ │ └── user.js
│ │ ├── parsers/
│ │ │ ├── content.js
│ │ │ ├── discovery.js
│ │ │ ├── feed.js
│ │ │ ├── language.js
│ │ │ └── og.js
│ │ ├── server.js
│ │ ├── utilities/
│ │ │ ├── collections.js
│ │ │ ├── email.js
│ │ │ ├── merge.js
│ │ │ ├── random.js
│ │ │ ├── rate-limiter.js
│ │ │ ├── social.js
│ │ │ ├── upsert.js
│ │ │ ├── url.js
│ │ │ └── validation.js
│ │ ├── utils.js
│ │ └── workers/
│ │ ├── conductor.js
│ │ ├── og.js
│ │ ├── podcast.js
│ │ ├── rss.js
│ │ ├── social.js
│ │ └── stream.js
│ └── test-entry.js
├── app/
│ ├── electron-builder.yaml
│ ├── package.json
│ ├── public/
│ │ ├── actions.js
│ │ ├── electron.js
│ │ ├── entitlements.mac.plist
│ │ ├── entitlements.mas.plist
│ │ ├── favicons/
│ │ │ └── icon.icns
│ │ ├── index.html
│ │ ├── info.plist
│ │ ├── manifest.json
│ │ └── preload.js
│ ├── src/
│ │ ├── App.js
│ │ ├── AppRouter.js
│ │ ├── AuthedRoute.js
│ │ ├── UnauthedRoute.js
│ │ ├── api/
│ │ │ ├── folderAPI.js
│ │ │ ├── index.js
│ │ │ ├── noteAPI.js
│ │ │ └── tagAPI.js
│ │ ├── components/
│ │ │ ├── AddOPMLModal.js
│ │ │ ├── AddPodcastModal.js
│ │ │ ├── AddRSSModal.js
│ │ │ ├── AliasModal.js
│ │ │ ├── AllArticlesList.js
│ │ │ ├── AllEpisodesList.js
│ │ │ ├── ArticleListItem.js
│ │ │ ├── Avatar/
│ │ │ │ └── index.js
│ │ │ ├── BookmarkPanel.js
│ │ │ ├── Drawer.js
│ │ │ ├── EpisodeListItem.js
│ │ │ ├── FeaturedItems.js
│ │ │ ├── FeedHeader.js
│ │ │ ├── FeedListItem.js
│ │ │ ├── Folder/
│ │ │ │ ├── DeleteModal.js
│ │ │ │ ├── FeedToFolderModal.js
│ │ │ │ ├── Folder.js
│ │ │ │ ├── FolderFeeds.js
│ │ │ │ ├── FolderPanel.js
│ │ │ │ ├── FolderPopover.js
│ │ │ │ ├── IntroFolders.js
│ │ │ │ ├── NewFolderModal.js
│ │ │ │ ├── RenameModal.js
│ │ │ │ └── SearchFeed.js
│ │ │ ├── Header.js
│ │ │ ├── HtmlRender.js
│ │ │ ├── Loader.js
│ │ │ ├── MediaCard.js
│ │ │ ├── Notes/
│ │ │ │ ├── HighlightMenu.js
│ │ │ │ ├── NoteInput.js
│ │ │ │ └── RecentNotesPanel.js
│ │ │ ├── Panel.js
│ │ │ ├── Player.js
│ │ │ ├── PodcastEpisode.js
│ │ │ ├── PodcastEpisodesView.js
│ │ │ ├── PodcastPanels/
│ │ │ │ ├── PodcastList.js
│ │ │ │ ├── RecentEpisodesPanel.js
│ │ │ │ ├── SuggestedPodcasts.js
│ │ │ │ └── index.js
│ │ │ ├── RSSArticle.js
│ │ │ ├── RSSArticleList.js
│ │ │ ├── RSSPanels/
│ │ │ │ ├── RecentArticlesPanel.js
│ │ │ │ ├── RssFeedList.js
│ │ │ │ ├── SuggestedFeeds.js
│ │ │ │ └── index.js
│ │ │ ├── SearchBar.js
│ │ │ ├── SimpleProgressBar.js
│ │ │ ├── Tabs.js
│ │ │ ├── Tag/
│ │ │ │ ├── DeleteModal.js
│ │ │ │ ├── RenameModal.js
│ │ │ │ ├── Tag.js
│ │ │ │ ├── TagFeeds.js
│ │ │ │ ├── TagPanel.js
│ │ │ │ └── TagView.js
│ │ │ ├── TimeAgo/
│ │ │ │ └── index.js
│ │ │ └── UserProfileSettingsDrawer.js
│ │ ├── config.js
│ │ ├── index.js
│ │ ├── reducers.js
│ │ ├── serviceWorker.js
│ │ ├── static-data/
│ │ │ └── onboarding-topics.js
│ │ ├── styles/
│ │ │ ├── base/
│ │ │ │ ├── _colors.scss
│ │ │ │ ├── _normalize.scss
│ │ │ │ ├── _typography.scss
│ │ │ │ └── _vars.scss
│ │ │ ├── components/
│ │ │ │ ├── _click-catcher.scss
│ │ │ │ ├── _columns.scss
│ │ │ │ ├── _comment-input-box.scss
│ │ │ │ ├── _comment-section.scss
│ │ │ │ ├── _content.scss
│ │ │ │ ├── _episode-info-popover.scss
│ │ │ │ ├── _feed-header.scss
│ │ │ │ ├── _feed-list-item.scss
│ │ │ │ ├── _follow-popover.scss
│ │ │ │ ├── _github.scss
│ │ │ │ ├── _hero-card.scss
│ │ │ │ ├── _item-info.scss
│ │ │ │ ├── _list.scss
│ │ │ │ ├── _loader.scss
│ │ │ │ ├── _media-card.scss
│ │ │ │ ├── _panel-element.scss
│ │ │ │ ├── _playlist-card.scss
│ │ │ │ ├── _popover-panel.scss
│ │ │ │ ├── _shows-grid.scss
│ │ │ │ ├── _simple-progress-bar.scss
│ │ │ │ └── _tiny-list.scss
│ │ │ ├── elements/
│ │ │ │ ├── _buttons.scss
│ │ │ │ ├── _drawer.scss
│ │ │ │ ├── _forms.scss
│ │ │ │ ├── _modals.scss
│ │ │ │ ├── _popovers.scss
│ │ │ │ └── _tabs.scss
│ │ │ ├── framework/
│ │ │ │ ├── _app.scss
│ │ │ │ ├── _player.scss
│ │ │ │ ├── auth-views/
│ │ │ │ │ ├── _create.scss
│ │ │ │ │ ├── _forgot-password.scss
│ │ │ │ │ ├── _login.scss
│ │ │ │ │ ├── _reset-password.scss
│ │ │ │ │ └── _shared.scss
│ │ │ │ └── dashboard/
│ │ │ │ └── _header.scss
│ │ │ ├── global.scss
│ │ │ ├── modules/
│ │ │ │ ├── _activity-feed.scss
│ │ │ │ ├── _add-content.scss
│ │ │ │ ├── _download.scss
│ │ │ │ ├── _featured-items-section.scss
│ │ │ │ ├── _follow-suggestions.scss
│ │ │ │ ├── _my-playlists.scss
│ │ │ │ ├── _notification-feed.scss
│ │ │ │ ├── _podcast-suggestions.scss
│ │ │ │ ├── _reshare-modal.scss
│ │ │ │ ├── _rss-panels.scss
│ │ │ │ ├── _search-results.scss
│ │ │ │ ├── _share.scss
│ │ │ │ ├── _social-icons.scss
│ │ │ │ └── _user-settings-drawer.scss
│ │ │ └── views/
│ │ │ ├── _404.scss
│ │ │ ├── _admin.scss
│ │ │ ├── _dashboard.scss
│ │ │ ├── _folder.scss
│ │ │ ├── _grid.scss
│ │ │ ├── _note.scss
│ │ │ ├── _onboarding.scss
│ │ │ ├── _playlist.scss
│ │ │ ├── _podcast.scss
│ │ │ ├── _profile.scss
│ │ │ └── _rss.scss
│ │ ├── util/
│ │ │ ├── feeds.js
│ │ │ ├── fetch/
│ │ │ │ └── index.js
│ │ │ ├── getFeedActivities.js
│ │ │ ├── getPlaceholderImageURL.js
│ │ │ ├── pins.js
│ │ │ └── social.js
│ │ └── views/
│ │ ├── 404View.js
│ │ ├── AdminView.js
│ │ ├── Dashboard.js
│ │ ├── FoldersView.js
│ │ ├── PodcastsView.js
│ │ ├── RSSFeedsView.js
│ │ └── auth-views/
│ │ ├── Create.js
│ │ ├── ForgotPassword.js
│ │ ├── Login.js
│ │ ├── ResetPassword.js
│ │ └── index.js
│ └── stylelint.config.js
├── process_dev.json
└── scripts/
└── docker-cleanup.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .eslintrc.js
================================================
module.exports = {
env: {
'browser': true,
'es6': true,
'node': true,
'shared-node-browser': true,
'mocha': true,
},
extends: ['eslint:recommended', 'plugin:react/recommended'],
parser: 'babel-eslint',
parserOptions: {
ecmaFeatures: {
experimentalObjectRestSpread: true,
jsx: true,
},
sourceType: 'module',
},
plugins: ['react'],
rules: {
'indent': ['error', 'tab'],
'linebreak-style': ['error', 'unix'],
'jsx-quotes': ['error', 'prefer-double'],
'quotes': ['error', 'single'],
'semi': ['error', 'always'],
'comma-dangle': ['error', 'always-multiline'],
'no-case-declarations': 'off',
'react/jsx-sort-props': 'error',
'eqeqeq': 'warn',
'quote-props': ['warn', 'consistent-as-needed'],
'react/no-deprecated': 'off',
'no-console': 0,
'keyword-spacing': ['error']
},
};
================================================
FILE: .gitattributes
================================================
* linguist-vendored
*.js linguist-vendored=false
================================================
FILE: .github/ISSUE_TEMPLATE.md
================================================
<!--
Have you read Winds Code of Conduct? By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/GetStream/Winds/blob/master/CODE_OF_CONDUCT.md
Do you want to ask a question? Are you looking for support? Please email [support@getstream.io](mailto:support@getstream.io).
-->
### Prerequisites
* [ ] Put an X between the brackets on this line if you have done all of the following:
* Checked that your issue isn't already filed: https://github.com/GetStream/Winds/issues
### Description
[Description of the issue]
### Steps to Reproduce
1. [First Step]
2. [Second Step]
3. [and so on...]
**Expected behavior:** [What you expect to happen]
**Actual behavior:** [What actually happens]
**Reproduces how often:** [What percentage of the time does it reproduce?]
### Versions
You can get this information from the manu. Winds > About Winds.
### Additional Information
Any additional information, configuration or data that might be necessary to reproduce the issue.
================================================
FILE: .github/PULL_REQUEST_TEMPLATE.md
================================================
### Requirements
* Filling out the template is required. Any pull request that does not include enough information to be reviewed in a timely manner may be closed at the maintainers' discretion.
* All new code requires tests to ensure against regressions
### Description of the Change
<!--
We must be able to understand the design of your change from this description. If we can't get a good idea of what the code will be doing from the description here, the pull request may be closed at the maintainers' discretion. Keep in mind that the maintainer reviewing this PR may not be familiar with or have worked with the code here recently, so please walk us through the concepts.
-->
### Alternate Designs
<!-- Explain what other alternates were considered and why the proposed version was selected -->
### Why Should This Be in Core?
<!-- Explain why this functionality should be in GetStream/Winds as opposed to a package -->
### Benefits
<!-- What benefits will be realized by the code change? -->
### Possible Drawbacks
<!-- What are the possible side-effects or negative impacts of the code change? -->
### Applicable Issues
<!-- Enter any applicable Issues here -->
================================================
FILE: .github/stale.yml
================================================
# Number of days of inactivity before an issue becomes stale
daysUntilStale: 30
# Number of days of inactivity before a stale issue is closed
daysUntilClose: 7
# Issues with these labels will never be considered stale
exemptLabels:
- pinned
- security
# Label to use when marking an issue as stale
staleLabel: wontfix
# Comment to post when marking an issue as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
# Comment to post when closing a stale issue. Set to `false` to disable
closeComment: true
================================================
FILE: .gitignore
================================================
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# Coverage report for codecov
coverage.lcov
.pyc
__pycache__/
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Typescript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.*
# System
.DS_Store
# Electron Build
#app-update.yml
#dev-app-update.yml
# Sass support
src/**/*.css
app/src/styles/global.css
# API & Worker Production Builds
api/dist
workers/dist
# App Production Builds
app/dist
app/build
app/assets/private-key.p12
# Fabric
fabfile.py
fabfile.pyc
# Redis
dump.rdb
# local mongo
mongo-db
# Deployment Configurations
deploy.sh
process_prod.json
embedded.provisionprofile
sign.js
*.p12
app/latest.html
#IDE
.vscode
================================================
FILE: .prettierrc
================================================
{
"useTabs": true,
"printWidth": 90,
"tabWidth": 4,
"singleQuote": true,
"trailingComma": "all",
"jsxBracketSameLine": false,
"parser": "babel",
"semi": true,
"arrowParens": "always"
}
================================================
FILE: .travis.yml
================================================
dist: xenial
language: node_js
node_js:
- "14"
services:
- redis
- mongodb
cache: yarn
install:
- "cd api && yarn install"
script:
- "yarn run test"
after_success:
- "yarn run coverage"
================================================
FILE: Dockerfile
================================================
# Use the latest version of Node
FROM mhart/alpine-node:latest
# Update dependency cache
RUN apk update && apk upgrade
# install dependencies
RUN apk add --no-cache make gcc g++ python git
# Install PM2 globally
RUN yarn global add pm2
# Create app directory
WORKDIR /usr/src/winds
# Copy app source code
COPY . .
# Expose port 8080
EXPOSE 8080
# Run process via pm2
CMD ["pm2-runtime", "start", "process_prod.json"]
================================================
FILE: LICENSE
================================================
Copyright 2018 Stream.io Inc
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the author nor the names of contributors may be used to
endorse or promote products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: README.md
================================================
> 🛑 **Notice**: This repository is no longer maintained; No further Issues or Pull Requests will be considered or approved.
# Winds - A Beautiful Open Source RSS & Podcast App Powered by GetStream.io
[](https://communityinviter.com/apps/winds-community-hq/winds-2-0)
[](https://travis-ci.org/GetStream/Winds)
[](https://codecov.io/gh/GetStream/Winds)
[](https://shields.io/)
[](https://github.com/GetStream/winds/graphs/commit-activity)
[](httpds://shields.io/)
[](https://stackshare.io/stream/winds)
## Description
Winds is a beautiful open-source RSS and Podcast app created using React & Redux on the frontend and Express.js on the backend. Use the free hosted version or run it on your own server and customize it as you see fit. Contributions in form of pull requests are always appreciated. Activity Feeds & Discovery in Winds are powered by [Stream](https://getstream.io/get_started/), the app leverages [Algolia](https://algolia.com?ref=stream) for search, [AWS](https://aws.amazon.com/) for hosting, [MongoDB Atlas](http://mbsy.co/mongodb/228644) for a hosted database (DBaaS), and [SendGrid](https://sendgrid.com/) for email. All of these services have a free tier.
## Getting Started
To get started with Winds, please download [the latest release](https://s3.amazonaws.com/winds-2.0-releases/latest.html)
## Featured RSS & Podcasts
Have a popular RSS or Podcast and want to be featured? Reach out to winds@getstream.io. We reply to every message.
# Features at a Glance
Winds is packed full of awesome features behind a beautiful user interface and user experience. Here's a quick look at what the project has to offer:
## Beautiful UI
## RSS & Podcast Recommendations
## Integrated Search
## Podcast Player
## RSS Reader
## TOCd
* [Roadmap](#roadmap)
* [Powered By](#powered-by)
* [Stream](#stream)
* [Algolia](#algolia)
* [MongoDB](#mongodb)
* [SendGrid](https://sendgrid.com)
* [AWS](https://aws.amazon.com/)
* [Tutorials](#tutorials)
* [Download](#download)
* [Contributing to Winds](#contributing-to-winds)
* [Support](#support)
* [Maintenance & Contributions](#maintenance-and-contributions)
## Roadmap
Help us improve Winds and/or vote on the [Roadmap for 2.1](https://github.com/GetStream/Winds/issues/191)
* [ ] Search detail screen
* [ ] Playlist support (partially implemented)
* [ ] Team support (share an activity feed with colleagues or friends to discover and collaborate)
* [ ] Mobile application powered by React Native
## Powered By
1. [Express](https://expressjs.com?ref=winds)
2. [React](https://reactjs.org?ref=winds) & [Redux](https://redux.js.org?ref=winds)
3. [Algolia](https://www.algolia.com?ref=winds)
4. [MongoDB Atlas](http://mbsy.co/mongodb/228644)
5. [SendGrid](https://sendgrid.com?ref=winds)
6. [Bull](https://github.com/OptimalBits/bull?ref=winds)
7. [Mercury](https://mercury.postlight.com?ref=winds)
8. [Stream](https://getstream.io?ref=winds)
9. [Sentry](https://sentry.io/?ref=winds)
10. [AWS](https://aws.amazon.com/?ref=winds)
**The full stack can be found on [StackShare.io](https://stackshare.io/stream/winds).**
### Stream
[Stream](https://getstream.io/?ref=winds) is an API for building activity feeds. For Winds the follow suggestions and the list of articles from the feeds you follow is powered by Stream. Stream accounts are free for up to 3 million feed updates and handle personalization (machine learning) for up to 100 users.
### Algolia
[Algolia](https://algolia.com?ref=winds) is used for lightning fast and relevant search. We use their [JavaScript search client](https://www.npmjs.com/package/algoliasearch?ref=winds) to easily setup the Winds search implementation. Algolia, accounts are free up to 10k records and 100k operations.
### MongoDB
[MongoDB Atlas](http://mbsy.co/mongodb/228644) provides a Database as a Service, and serves as the backend datastore for Winds.
## Tutorials & Blog Posts
The following tutorials will not only help you start contributing to Winds, but also provide inspiration for your next app.
**Note:** We're actively working on this portion of the README. To stay up to date with the latest information, please signup for the hosted version at [https://getstream.io/winds](https://getstream.io/winds).
1. [Implementing search with Algolia](https://hackernoon.com/integrating-algolia-search-in-winds-a-beautiful-rss-podcast-application-f231e49cdab5)
2. [Stream and Winds](https://getstream.io/blog/the-engine-that-powers-winds/)
3. [Running PM2 & Node.js in Production Environments](https://hackernoon.com/running-pm2-node-js-in-production-environments-13e703fc108a)
4. [Creating a RESTful API design with Express.js](https://hackernoon.com/building-a-node-js-powered-api-with-express-mongoose-mongodb-19b14fd4b51e)
5. [Takeaways on Building a React Based App with Electron](https://medium.com/@nparsons08/publishing-a-react-based-app-with-electron-and-nodejs-f5ec44169366)
6. [The Winds Stack](https://stackshare.io/stream/how-stream-built-a-modern-rss-reader-with-javascript)
7. [Building Touch Bar Support for macOS in Electron with React](https://hackernoon.com/winds-2-1-building-touch-bar-support-for-macos-in-electron-with-react-e10adb811c91)
8. [Testing Node.js in 2018](https://hackernoon.com/testing-node-js-in-2018-10a04dd77391)
9. [Simple Steps to Optimize Your App Performance with MongoDB, Redis, and Node.js](https://hackernoon.com/simple-steps-to-optimize-your-app-performance-5700d8b58f58)
10. [Getting Started with Winds & Open Source](https://hackernoon.com/winds-an-in-depth-tutorial-on-making-your-first-contribution-to-open-source-software-ebf259f21db2)
11. [Deploying the Winds App to Amazon S3 and CloudFront](https://getstream.io/blog/deploying-the-winds-app-to-amazon-s3-and-cloudfront/)
12. [Deploying the Winds API to AWS ECS with Docker Compose](https://getstream.io/blog/deploying-the-winds-api-to-aws-ecs-with-docker-compose/)
## Download
To download Winds, visit [https://getstream.io/winds/](https://getstream.io/winds/).
## Contributing to Winds
### TL;DR
Commands:
* `brew install redis mongodb`
* `brew services start mongodb`
* `brew services start redis`
* `cd Winds`
* `cd api && yarn`
* `cd ../app && yarn`
Sign up for both Stream and Algolia, and create the following `.env` file in the `app` directory, replacing the keys where indicated:
```
DATABASE_URI=mongodb://localhost/WINDS_DEV
CACHE_URI=redis://localhost:6379
JWT_SECRET=YOUR_JWT_SECRET
API_PORT=8080
REACT_APP_API_ENDPOINT=http://localhost:8080
STREAM_API_BASE_URL=https://windspersonalization.getstream.io/personalization/v1.0
STREAM_APP_ID=YOUR_STREAM_APP_ID
REACT_APP_STREAM_APP_ID=YOUR_STREAM_APP_ID
REACT_APP_STREAM_API_KEY=YOUR_STREAM_API_KEY
REACT_APP_STREAM_ANALYTICS=YOUR_STREAM_ANALYTICS_TOKEN
STREAM_API_KEY=YOUR_STREAM_API_KEY
STREAM_API_SECRET=YOUR_STREAM_API_SECRET
REACT_APP_ALGOLIA_APP_ID=YOUR_ALGOLIA_APP_ID
REACT_APP_ALGOLIA_SEARCH_KEY=YOUR_ALGOLIA_SEARCH_ONLY_API_KEY
ALGOLIA_WRITE_KEY=YOUR_ALGOLIA_ADMIN_API_KEY
```
> Note: If you are running the test suite, you will need to have a test version of the `.env` file inside of the `api/test` directory.
Then run:
* `pm2 start process_dev.json`
* `cd app && yarn start`
### Clone the Repo
```bash
git clone git@github.com:GetStream/Winds.git
```
### Install dependencies
The following instructions are geared towards Mac users who can use `brew` ([Homebrew](https://brew.sh/)) to install most dependencies. Ubuntu users can use `apt`, and Windows users will have to install directly from the dependency's site. Non-debian-based Linux users will probably be able to figure it out on their own :)
* `cd Winds/app`
* `yarn`
* `cd ../api`
* `yarn`
### Start MongoDB Locally
Winds uses MongoDB as the main datastore - it contains all users, rss feeds, podcasts, episodes, articles, and shares.
If you're on a Mac, you can install MongoDB through [Homebrew](https://brew.sh/) by running:
```
brew install mongodb
```
_(You can also install MongoDB from the [official MongoDB site](https://www.mongodb.com/download-center).)_
You can also run MongoDB in the background by running:
```
brew services start mongodb
```
### Start Redis Locally
At Stream, we use Redis as an in-memory storage for the Winds podcast processing and RSS processing workers. It contains a list of podcasts and RSS feeds, which the workers pick up and process using the `bull` messaging system.
If you're on a Mac, you can install Redis through [Homebrew](https://brew.sh/) by running:
```
brew install redis
```
_(You can also install Redis from the [official Redis site](https://redis.io/download).)_
Then, start Redis by running:
```
redis-server
```
...which creates (by default) a `dump.rdb` file in your current directory and stores a cached version of the database in that location.
You can also run Redis in the background by running:
```
brew services start redis
```
### Loading Test Data
For testing purposes, you will want to use the test data located [here](https://s3.amazonaws.com/winds-hosted/static/export/WINDS.zip).
Use [`mongoimport`](https://docs.mongodb.com/manual/reference/program/mongoimport/) or [`mongorestore`](https://docs.mongodb.com/manual/reference/program/mongorestore/) to import the data. There are two username and password combinations for testing:
**Username**: `admin@admin.com`<br/>
**Password**: `admin`
<br/><br/>
**Username**: `test@test.com`<br/>
**Password**: `test`
You will need to run the `FLUSHALL` command in Redis to ensure that the new content is picked up.
> Note: This will override any local data that you may have. Please be cautious! Also, this will not create Stream follows – please follow feeds manually to generate them.
### Stream
#### Sign up and Create a Stream App
To contribute to Winds, sign up for [Stream](https://getstream.io/get_started?ref=winds) to utilize the activity and timeline feeds.
_(Reminder: Stream is free for applications with less than 3,000,000 updates per month.)_
* [Sign up for Stream here](https://getstream.io/get_started?ref=winds)
* Create a new Stream app
* Find the App ID, API Key, and API Secret for your new Stream app
#### Add your Stream App ID, API Key, and API Secret to your `.env`
Append the Stream App ID, API Key, and API secret to your `.env` file:
```
STREAM_APP_ID=YOUR_STREAM_APP_ID
STREAM_API_KEY=YOUR_STREAM_API_KEY
STREAM_API_SECRET=YOUR_STREAM_API_SECRET
```
#### Create Your Stream Feed Groups
Once you've signed in, create "feed groups" for your Stream app.
A "feed group" defines a certain type of feed within your application. Use the "Add Feed Group" button to create the following feeds:
| Feed Group Name | Feed Group Type |
| --------------- | --------------- |
| `podcast` | flat |
| `rss` | flat |
| `user` | flat |
| `timeline` | flat |
| `folder` | flat |
| `user_episode` | flat |
| `user_article` | flat |
It's fine to enable "Realtime Notifications" for each of these feed groups, though we won't be using those yet.
### Algolia
#### Sign up for Algolia and Create an Algolia App and Index
In addition to Stream, you also need to sign up for [Algolia](https://www.algolia.com/users/sign_up?ref=winds), to contribute to Winds, for the search functionality.
_(Algolia is free for applications with up to 10k records.)_
* [Sign up for Algolia here](https://www.algolia.com/users/sign_up?ref=winds)
* From the [Applications page](https://www.algolia.com/manage/applications), click "New Application" and create a new Algolia application. (We recommend something similar to `my-winds-app`)
* (Select the datacenter that's closest to you.)
* From the application page, select "Indices" and click "Add New Index". (We recommend something similar to `winds-main-index`)
#### Add Your Algolia Application Id, Search-Only Api Key and Admin Api Key to Your `.env` File
From your app, click the "API Keys" button on the left to find your app ID and API keys.
Append your Algolia application ID, search-only API Key and Admin API Key to your `.env` file to look like this:
```
REACT_APP_ALGOLIA_APP_ID=YOUR_ALGOLIA_APP_ID
REACT_APP_ALGOLIA_SEARCH_KEY=YOUR_ALGOLIA_SEARCH_ONLY_API_KEY
ALGOLIA_WRITE_KEY=YOUR_ALGOLIA_ADMIN_API_KEY
```
### Start Backend Services
From the root directory, run:
```
pm2 start process_dev.json
```
To see logging information for all processes, run:
```
pm2 logs
```
### Start Frontend Electron / Web App Server
```
cd app && yarn start
```
### Running tests
Winds API server uses:
* [Mocha](https://mochajs.org) as testing framework
* [Chai](https://chaijs.org) as assertion library
* [Sinon](https://sinonjs.org) as mocking library
* [nock](https://github.com/node-nock/nock) as HTTP mocking library
* [mock-require](https://github.com/boblauer/mock-require) as module mocking library
Tests are located in [`api/test` folder](https://github.com/GetStream/Winds/tree/master/api/test).
File structure is intentionally mirroring files in `api/src` to simplify matching tests to tested code.
To run tests:
```
cd api && yarn run test
```
To run tests with extended stack traces (useful when debugging async issues):
```
cd api && yarn run test_deep
```
#### Adding new tests
Add your code to a file in `api/test` folder (preferably mirroring existing file from `api/src` folder).
Refer to [Mocha documentation](https://mochajs.org/#getting-started) for examples of using BDD-like DSL for writing tests.
Modules are mocked in [`api/test-entry.js`](https://github.com/GetStream/Winds/blob/master/api/test-entry.js#L21L27) as mocks have to be installed before any modules are loaded.
Fixtures are loaded via [`loadFixture`](https://github.com/GetStream/Winds/blob/master/api/test/utils.js#L59L101) function from [`api/test/fixtures` folder](https://github.com/GetStream/Winds/tree/master/api/test/fixtures)
Various utility functions are provided in [`api/test/util.js`](https://github.com/GetStream/Winds/blob/master/api/test/utils.js) (See other tests for examples of usage).
### Building a Production Version
Build a production version of Winds by running from root directory:
```
./api/build.sh
```
This creates production-ready JS files in api/dist.
To run the production JS files:
```
pm2 start process_prod.json
```
OR
**Prepare the build for Docker**:
`cd api && cd scripts && ./make-build.sh`
**Build the Docker container (API & all workers)**:
`cd ../ && docker-compose up`
The commands above will prepare and start the API (and all workers). The frontend will still need to be started manually.
## Debugging RSS & Podcast Issues
Unfortunately there is no unified standard for RSS.
Go to the `api` directory and run `yarn link` to make these commands available:
```
winds rss https://techcrunch.com/feed/
```
**Podcasts**:
```
winds podcast https://www.npr.org/rss/podcast.php\?id\=510289
```
**Open Graph scraping**:
```
winds og http://www.planetary.org/multimedia/planetary-radio/show/2018/0509-amy-mainzer-neowise.html
```
**RSS Discovery**:
```
winds discover mashable.com
```
**Article parsing (w/ Mercury)**:
```
winds article https://alexiskold.net/2018/04/12/meet-12-startups-from-techstars-nyc-winter-2018-program/
```
Pull requests for improved RSS compatibility are much appreciated.
Most of the parsing codebase is located in `api/src/parsers/`.
## Support
All support is handled via [GitHub Issues](https://github.com/getstream/winds/issues). If you're unfamiliar with creating an Issue on GitHub, please follow [these instructions](https://help.github.com/articles/creating-an-issue/).
## Maintenance and Contributions
Thank you to all of the maintainers and contributors who've helped Winds become what it is today and help it stay up and running every day. We couldn't do it without you!
### Special Shoutouts To:
* [Hackernoon](https://hackernoon.com/)
* [Product Hunt](https://www.producthunt.com/)
* [StackShare](https://stackshare.io/stream/how-stream-built-a-modern-rss-reader-with-javascript)
### Primary Maintainers
* [Nick Parsons](https://github.com/nparsons08)
* [Amin Mahboubi](https://github.com/mahboubii)
* [Thierry Schellenbach](https://github.com/tschellenbach)
* [Josh Tilton](https://github.com/tilton)
### Contributors
* [Tommaso Barbugli](https://github.com/tbarbugli)
* [Ken Hoff](https://github.com/kenhoff)
* [Dwight Gunning](https://github.com/dwightgunning)
* [Matt Gauger](https://github.com/mathias)
* [Max Klyga](https://github.com/nekuromento)
* [Zhomart Mukhamejanov](https://github.com/Zhomart)
* [Julian Xhokaxhiu](https://github.com/julianxhokaxhiu)
* [Jonathon Belotti](https://github.com/thundergolfer)
* [The Gitter Badger](https://github.com/gitter-badger)
* [Meriadec Pillet](https://github.com/meriadec)
* [Alex Sinnott](https://github.com/sifex)
* [Lawal Sauban](https://github.com/sauban)
## Revive RSS
RSS is an amazing open standard. It is probably the most pleasant way to stay up to date with the sites and podcasts you care about. Our reasons for contributing to Winds are explained in the blogpost [Winds 2.0 It's time to Revive RSS](https://getstream.io/blog/winds-2-0-its-time-to-revive-rss/). In this section we will list other open source and commercial projects that are having an impact on Reviving RSS:
* [Miniflux](https://github.com/miniflux/miniflux)
* [TwitRSSMe](https://twitrss.me/)
* [Feedly](https://feedly.com/)
* [NewsBlur](https://newsblur.com/)
* [Feedity](https://feedity.com/)
* [SaveRSS](https://mg.guelker.eu/saverss/)
## We are hiring!
We've recently closed a [$38 million Series B funding round](https://techcrunch.com/2021/03/04/stream-raises-38m-as-its-chat-and-activity-feed-apis-power-communications-for-1b-users/) and we keep actively growing.
Our APIs are used by more than a billion end-users, and you'll have a chance to make a huge impact on the product within a team of the strongest engineers all over the world.
Check out our current openings and apply via [Stream's website](https://getstream.io/team/#jobs).
================================================
FILE: RSS.md
================================================
## Post Uniqueness ##
Post uniqueness in an RSS feed can be determined by 4 different methods:
- The guid property on the post (not always present)
- The link property on the post (not always present and sometimes uses the site url)
- The url of the first enclosure (common amongst podcasts)
- A hash of the title, link, description and enclosures
Note that the guid shouldn't change. The link, url and hash can change when the post is updated though.
So using the first approach is preferable.
While not available in the RSS feed you could also consider the
- Canonical URL on the page
## RSS feed Uniqueness ##
Determining the uniqueness for an RSS feed is harder.
There are a few different options
- A normalized version of the feed url
- The feed url specified in the RSS doc
Neither of those approaches work as most RSS feeds are available under many different urls.
- A hash based on the last 10 article hashes
## Discovery ##
The standard discovery library picks up the link rel tag in the html.
Many sites have dropped support for this tag though. We could add special cases for
- YouTube
- Wordpress blogs
As it's easy to determine the feed location
## How Winds handles uniqueness ##
For every feed Winds will evaluate which one of these fields are unique:
['guid', 'link', 'enclosure[0].url', 'hash']
Note that the hash is computed before any enrichment is done on the feed content.
After that it stores the unique value in `article.fingerprint` in the format `guid:123` or `hash:123` etc.
After selecting the best algorithm it will use a batch select and update to update the feed articles.
The uniqueness of the last 20 articles is used to compute a hash for the RSS feed.
We use this information to occasionally merge RSS feeds.
After merging the alternative URLs are stored to prevent people from submitting the same feed under a different url.
================================================
FILE: STYLE.md
================================================
The style rules for Winds are defined in:
* .prettierrc
* .eslintrc
To cleanup your code run
`yarn prettier` in the api directory
If you're using atom be sure to install prettier-atom and make sure it finds our prettierrc config file
================================================
FILE: api/Dockerfile
================================================
# Use the latest version of Node
FROM mhart/alpine-node:latest
# Update dependency cache
RUN apk update && apk upgrade
# install dependencies
RUN apk add --no-cache make gcc g++ python git
# Install PM2 globally
RUN yarn global add pm2
# Create app directory
WORKDIR /usr/src/api
# Copy package.json for build
COPY package.json ./
# Copy app source code
COPY . .
# Expose port 8080
EXPOSE 8080
# Run process via pm2
CMD ["pm2-runtime", "start", "process_prod.json"]
================================================
FILE: api/build.sh
================================================
#!/bin/bash
cd api
# Remove the existing build directory and create a fresh one
rm -rf dist && mkdir -p dist/{utils,email/templates}
# Transpile ES6 to JavaScript
npx babel src --out-dir dist --ignore node_modules
# Copy build files to fresh /dist directory
cp package.json dist/package.json
# Copy email files to fresh /dist directory
cp -R src/utils/email/templates dist/utils/email
# Install node modules via yarn
cd dist && yarn install --production --modules-folder node_modules
================================================
FILE: api/config.yaml
================================================
apiVersion: v1
kind: Service
metadata:
name: api
annotations:
cloud.google.com/load-balancer-type: "Internal"
labels:
app: api
spec:
type: LoadBalancer
ports:
- port: 443
protocol: TCP
selector:
api: api
================================================
FILE: api/docker-compose.yml
================================================
version: '3.7'
services:
api:
build:
context: .
dockerfile: Dockerfile
volumes:
- '/opt/data:/opt/data'
ports:
- '8080:8080'
depends_on:
- cache
- database
links:
- cache
- database
networks:
- backend
logging:
driver: json-file
options:
max-size: 100MB
max-file: '3'
environment:
NODE_ENV: production
DOCKER: 'true'
PRODUCT_URL: 'https://getstream.io/winds'
PRODUCT_NAME: Winds
PRODUCT_AUTHOR: Stream
DATABASE_URI: 'mongodb://database/WINDS'
CACHE_URI: 'redis://cache:6379'
JWT_SECRET: INSERT_JWT_SECRET_HERE
API_PORT: 8080
STREAM_API_BASE_URL: 'https://windspersonalization.getstream.io/personalization/v1.0'
STREAM_APP_ID: INSERT_STREAM_API_ID_HERE
STREAM_API_KEY: INSERT_STREAM_API_KEY_HERE
STREAM_API_SECRET: INSERT_STREAM_API_SECRET_HERE
ALGOLIA_WRITE_KEY: INSERT_ALGOLIA_WRITE_KEY_HERE
cache:
image: 'redis:alpine'
ports:
- '6379:6379'
networks:
- backend
restart: always
logging:
driver: json-file
options:
max-size: 100MB
max-file: '3'
database:
image: mongo
ports:
- '27017:27017'
networks:
- backend
logging:
driver: json-file
options:
max-size: 100MB
max-file: '3'
networks:
backend: null
================================================
FILE: api/ecosystem.dev.config.js
================================================
module.exports = {
apps: [
{
name: 'api',
interpreter: 'babel-node',
script: 'src/server.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
{
name: 'conductor',
interpreter: 'babel-node',
script: 'src/workers/conductor.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
{
name: 'stream',
interpreter: 'babel-node',
script: 'src/workers/stream.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
{
name: 'rss',
interpreter: 'babel-node',
script: 'src/workers/rss.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
{
name: 'podcast',
interpreter: 'babel-node',
script: 'src/workers/podcast.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
{
name: 'og',
interpreter: 'babel-node',
script: 'src/workers/og.js',
watch: true,
ignore_watch: ['.git', 'node_modules'],
},
],
};
================================================
FILE: api/now.json
================================================
{
"type": "docker",
"features": {
"cloud": "v2"
}
}
================================================
FILE: api/package.json
================================================
{
"name": "api",
"description": "https://getstream.io/winds",
"license": "BSD-3-Clause",
"scripts": {
"preinstall": "yarn global add pm2",
"start": "cd dist && node server.js",
"build": "babel ./src --out-dir ./dist",
"watch": "nodemon start",
"dev": "pm2 start ecosystem.dev.config.js && pm2 log",
"prettier": "prettier --config ../.prettierrc --write \"src/**/*.js\"",
"test": "NODE_ENV=test node --max-old-space-size=16384 --optimize-for-size node_modules/nyc/bin/nyc.js node_modules/mocha/bin/_mocha --timeout 15000 --require test-entry.js \"test/**/*.js\"",
"coverage": "NODE_ENV=test node_modules/nyc/bin/nyc.js report --reporter=text-lcov > coverage.lcov && node_modules/codecov/bin/codecov",
"test_deep": "NODE_ENV=test node --max-old-space-size=16384 --optimize-for-size --stack_trace_limit=200 -r trace -r clarify node_modules/mocha/bin/_mocha --timeout 45000 --require test-entry.js \"test/**/*.js\""
},
"bin": {
"winds": "src/commands/winds.js"
},
"nyc": {
"sourceMap": false,
"instrument": false
},
"author": "Winds Team <winds@getstream.io>",
"keywords": [
"Winds",
"RSS",
"RSS Reader",
"Podcast",
"Podcast Player"
],
"engines": {
"node": ">=13"
},
"dependencies": {
"@postlight/mercury-parser": "^2.2.0",
"@sendgrid/mail": "^7.1.1",
"algoliasearch": "^4.2.0",
"body-parser": "^1.19.0",
"bull": "^3.20.0",
"bull-arena": "^3.7.0",
"compression": "^1.7.4",
"cors": "^2.8.5",
"deep-object-diff": "^1.1.0",
"dotenv": "^8.2.0",
"ejs": "^3.1.3",
"entities": "^1.1.2",
"express": "^4.17.1",
"express-basic-auth": "^1.2.0",
"express-jwt": "^5.3.3",
"express-rate-limit": "^5.1.3",
"favicon": "^0.0.2",
"feedparser": "^2.2.10",
"franc-min": "^5.0.0",
"getstream": "^4.5.1",
"gravatar": "^1.8.0",
"htmlparser2": "^4.1.0",
"inflate-auto": "^1.0.0",
"ioredis": "^4.19.4",
"joi": "^13.7.0",
"jsonwebtoken": "^8.5.1",
"moment": "^2.26.0",
"mongoose": "^5.9.15",
"mongoose-autopopulate": "^0.12.2",
"mongoose-bcrypt": "^1.8.0",
"mongoose-string-query": "^0.2.7",
"mongoose-timestamp": "^0.6.0",
"multer": "^1.4.2",
"node-opml-parser": "^1.0.0",
"node-statsd": "^0.1.1",
"normalize-url": "^5.0.0",
"opml-generator": "^1.1.1",
"progress": "^2.0.3",
"raven": "^2.6.4",
"request": "^2.88.2",
"request-promise-native": "^1.0.8",
"rss-finder": "^2.1.3",
"sanitize-html": "^1.24.0",
"stream-analytics": "^2.8.0",
"strip": "^3.0.0",
"uuid": "^8.1.0",
"validator": "^11.1.0",
"winston": "^3.2.1",
"winston-transport": "^4.3.0",
"yargs": "^15.3.1",
"zlib": "^1.0.5"
},
"devDependencies": {
"@babel/cli": "^7.8.4",
"@babel/core": "^7.9.6",
"@babel/node": "^7.8.7",
"@babel/preset-env": "^7.9.6",
"@babel/register": "^7.9.0",
"babel-plugin-istanbul": "^6.0.0",
"babel-plugin-shebang": "^1.0.0",
"chai": "^4.2.0",
"chai-http": "^4.3.0",
"clarify": "^2.1.0",
"codecov": "^3.7.0",
"flatted": "^2.0.2",
"mocha": "^7.2.0",
"mock-require": "^3.0.3",
"nock": "^10.0.6",
"nodemon": "^2.0.4",
"nyc": "^15.0.1",
"prettier": "^2.0.5",
"redis": "^2.8.0",
"sinon": "^7.3.2",
"trace": "^3.1.1"
}
}
================================================
FILE: api/scripts/docker-build.sh
================================================
#!/bin/bash
# Build the Docker image
docker build -t winds-api .
# List all Docker images
docker images
# Run the Docker image
docker run -p 8080:8080 -d winds-api
================================================
FILE: api/scripts/docker-compose-aws.sh
================================================
#!/bin/bash
# Build with Docker Compose
docker-compose -f ../docker-compose-aws.yml up
================================================
FILE: api/scripts/docker-compose.sh
================================================
#!/bin/bash
# Build with Docker Compose
docker-compose -f ../docker-compose.yml up
================================================
FILE: api/scripts/make-build.sh
================================================
#!/bin/bash
# Remove the existing build directory and create a fresh one
rm -rf ../dist && mkdir ../dist
# Transpile ES6 to JavaScript
npx babel ../src --out-dir ../dist --ignore node_modules
# Copy build files to fresh /dist directory
cp ../src/package.json ../dist/package.json
# Copy build files to fresh /dist directory
cp process_prod.json ../dist/process_prod.json
# Copy email files to fresh /dist directory
cp -R ../src/utils/email/templates ../dist/utils/email
# Install node modules via yarn
cd ../dist && yarn install --production --modules-folder node_modules
================================================
FILE: api/setup-tests.js
================================================
import fs from 'fs';
import path from 'path';
import Mocha from 'mocha';
import chai from 'chai';
import chaiHttp from 'chai-http';
import { stringify } from 'flatted/cjs';
import config from './src/config';
import api from './src/server';
import logger from './src/utils/logger';
import { dropDBs } from './test/utils';
api.use((err, req, res, next) => {
if (err) {
logger.error(err.stack);
if (err.request) {
logger.error(`REQUEST = ${stringify(err.request)}`);
}
if (err.response) {
logger.error(`RESPONSE = ${stringify(err.response)}`);
}
}
next(err, req, res);
});
chai.use(chaiHttp);
function wrapMocha(onPrepare, onUnprepare) {
// Monkey-patch run method
const run = Mocha.prototype.run;
//XXX: using function syntax instead of fat-arrow syntax
// to avoid implicit binding of 'this'
Mocha.prototype.run = function(done) {
const self = this;
onPrepare()
.then(() => {
//XXX: ditto
run.call(self, function() {
if (typeof onUnprepare === 'function') {
onUnprepare.apply(this, arguments);
}
done.apply(this, arguments);
});
})
.catch(err => {
if (err instanceof Error) {
console.error(err.stack);
}
process.exit(1);
});
};
}
wrapMocha(
async () => {
if (!config.database.uri)
throw new Error('Missing MongoDB connection string. Check config');
if (!config.cache.uri)
throw new Error('Missing Redis connection string. Check config');
if (
!config.database.uri.includes('localhost') &&
!config.database.uri.includes('127.0.0.1')
)
throw new Error(
'MongoDB connection string contains non-local address. For safety reasons test suite can only connect to local databases. Check config',
);
if (
!config.cache.uri.includes('localhost') &&
!config.cache.uri.includes('127.0.0.1')
)
throw new Error(
'Redis connection string contains non-local address. For safety reasons test suite can only connect to local databases. Check config',
);
//XXX: drop all data before running tests
await dropDBs();
fs.readdirSync(path.join(__dirname, 'src', 'routes')).forEach(file => {
if (file.endsWith('.js')) {
require(`./src/routes/${file}`)(api);
}
});
},
failures => {
//XXX: it seems Travis-ci is having trouble with process wrap-up procedures so lets
// allocate more time before shutting down
const timeout = process.env.TRAVIS ? 10000 : 1500;
logger.info(`Terminating in ${timeout / 1000} seconds`);
//XXX: don't care about open connections
setTimeout(() => process.exit(failures ? 1 : 0), timeout);
},
);
================================================
FILE: api/src/.babelrc
================================================
{
"plugins": [ "shebang"],
"presets": [["@babel/preset-env", { "targets": { "node": "current" } }]]
}
================================================
FILE: api/src/.prettierignore
================================================
config/index.js
================================================
FILE: api/src/asyncTasks.js
================================================
import config from './config';
import logger from './utils/logger';
import Queue from 'bull';
import { getStatsDClient } from './utils/statsd';
export const rssQueue = new Queue('rss', config.cache.uri, {
settings: {
lockDuration: 90000,
stalledInterval: 75000,
maxStalledCount: 2,
},
});
export const podcastQueue = new Queue('podcast', config.cache.uri, {
settings: {
lockDuration: 90000,
stalledInterval: 75000,
maxStalledCount: 2,
},
});
export const streamQueue = new Queue('stream', config.cache.uri, {
limiter: { max: 12000, duration: 3600000 }, // 12k per hour
});
export const ogQueue = new Queue('og', config.cache.uri, {
settings: {
lockDuration: 60000,
stalledInterval: 50000,
maxStalledCount: 2,
},
});
export const socialQueue = new Queue('social', config.cache.uri);
function makeMetricKey(queue, event) {
return ['winds', 'bull', queue.name, event].join('.');
}
async function trackQueueSize(statsd, queue) {
let queueStatus = await queue.getJobCounts();
statsd.gauge(makeMetricKey(queue, 'waiting'), queueStatus.waiting);
statsd.gauge(makeMetricKey(queue, 'active'), queueStatus.active);
}
function AddQueueTracking(queue) {
var statsd = getStatsDClient();
queue.on('error', function (err) {
statsd.increment(makeMetricKey(queue, 'error'));
logger.warn(
`Queue ${queue.name} encountered an unexpected error: ${err.message}`,
);
});
queue.on('active', function (job, jobPromise) {
statsd.increment(makeMetricKey(queue, 'active'));
});
queue.on('completed', function (job, result) {
statsd.timing(makeMetricKey(queue, 'elapsed'), new Date() - job.timestamp);
statsd.increment(makeMetricKey(queue, 'completed'));
});
queue.on('stalled', function (job) {
statsd.increment(makeMetricKey(queue, 'stalled'));
logger.warn(`Queue ${queue.name} job stalled: '${JSON.stringify(job)}'`);
});
queue.on('failed', function (job, err) {
statsd.increment(makeMetricKey(queue, 'failed'));
logger.warn(
`Queue ${queue.name} failed to process job '${JSON.stringify(job)}': ${
err.message
}`,
);
});
queue.on('paused', function () {
statsd.increment(makeMetricKey(queue, 'paused'));
});
queue.on('resumed', function (job) {
statsd.increment(makeMetricKey(queue, 'resumed'));
});
setInterval(trackQueueSize, 30000, statsd, queue);
}
const currentEnvironment = process.env.NODE_ENV || 'development';
if (currentEnvironment !== 'test') {
AddQueueTracking(rssQueue);
AddQueueTracking(ogQueue);
AddQueueTracking(podcastQueue);
AddQueueTracking(streamQueue);
AddQueueTracking(socialQueue);
}
export const RssQueueAdd = rssQueue.add.bind(rssQueue);
export const OgQueueAdd = ogQueue.add.bind(ogQueue);
export const PodcastQueueAdd = podcastQueue.add.bind(podcastQueue);
export const StreamQueueAdd = podcastQueue.add.bind(streamQueue);
export const SocialQueueAdd = podcastQueue.add.bind(socialQueue);
export function ProcessRssQueue() {
getStatsDClient().increment(makeMetricKey(rssQueue, 'started'));
return rssQueue.process(...arguments);
}
export function ProcessOgQueue() {
getStatsDClient().increment(makeMetricKey(ogQueue, 'started'));
return ogQueue.process(...arguments);
}
export function ProcessPodcastQueue() {
getStatsDClient().increment(makeMetricKey(podcastQueue, 'started'));
return podcastQueue.process(...arguments);
}
export function ProcessStreamQueue() {
getStatsDClient().increment(makeMetricKey(streamQueue, 'started'));
return streamQueue.process(...arguments);
}
export function ProcessSocialQueue() {
getStatsDClient().increment(makeMetricKey(socialQueue, 'started'));
return socialQueue.process(...arguments);
}
export function ShutDownRssQueue() {
getStatsDClient().increment(makeMetricKey(rssQueue, 'stopped'));
return socialQueue.close(...arguments);
}
export function ShutDownPodcastQueue() {
getStatsDClient().increment(makeMetricKey(podcastQueue, 'stopped'));
return socialQueue.close(...arguments);
}
export function ShutDownOgQueue() {
getStatsDClient().increment(makeMetricKey(ogQueue, 'stopped'));
return socialQueue.close(...arguments);
}
export function ShutDownSocialQueue() {
getStatsDClient().increment(makeMetricKey(socialQueue, 'stopped'));
return socialQueue.close(...arguments);
}
export function ShutDownStreamQueue() {
getStatsDClient().increment(makeMetricKey(streamQueue, 'stopped'));
return socialQueue.close(...arguments);
}
================================================
FILE: api/src/commands/_debug-feed.js
================================================
import program from 'commander';
import '../loadenv';
import '../utils/db';
import { ParseFeed, ParsePodcast } from '../parsers/feed';
import chalk from 'chalk';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import RSS from '../models/rss';
import { RssQueueAdd, PodcastQueueAdd } from '../asyncTasks';
import normalize from 'normalize-url';
// do stuff
export async function debugFeed(feedType, feedUrls) {
// This is a small helper tool to quickly help debug issues with podcasts or RSS feeds
logger.info(`Starting the ${feedType} Debugger \\0/`);
logger.info(
'Please report issues with RSS feeds here https://github.com/getstream/winds',
);
logger.info('Note that pull requests are much appreciated!');
logger.info(`Handling ${feedUrls.length} urls`);
for (let target of feedUrls) {
logger.info(`Looking up the first ${program.limit} articles from ${target}`);
async function validate(response) {
// validate the podcast or RSS feed
logger.info('========== Validating Publication ==========');
logger.info(`Title: ${response.title}`);
logger.info(`Link: ${response.link}`);
if (response.image && response.image.og) {
logger.info(chalk.green('Image found :)'));
logger.info(`Image: ${response.image.og}`);
} else {
logger.info(chalk.red('Image missing :('));
}
logger.info('========== Validating episodes/articles now ==========');
// validate the articles or episodes
let articles = response.articles ? response.articles : response.episodes;
let selectedArticles = articles.slice(0, program.limit);
logger.info(`Found ${articles.length} articles showing ${program.limit}`);
if (selectedArticles.length) {
for (let article of selectedArticles) {
logger.info('======================================');
logger.info(chalk.green(`Title: ${article.title}`));
logger.info(`URL: ${article.url}`);
logger.info(`Description: ${article.description}`);
logger.info(`Publication Date: ${article.publicationDate}`);
if (article.commentUrl) {
logger.info(`Comments: ${article.commentUrl}`);
}
if (article.content) {
logger.info(`Content: ${article.content}`);
}
// for RSS we rely on OG scraping, for podcasts the images are already in the feed
if (article.images && article.images.og) {
logger.info(chalk.green('Image found :)'));
logger.info(`Image: ${article.images.og}`);
} else {
logger.info(chalk.red('Image missing :('));
}
if (article.enclosures && article.enclosures.length) {
logger.info(`found ${article.enclosures.length} enclosures`);
for (let enclosure of article.enclosures) {
logger.info(JSON.stringify(enclosure));
}
}
if (feedType === 'podcast') {
if (article.enclosure) {
logger.info(chalk.green('Enclosure found :)'));
logger.info(article.enclosure);
} else {
logger.info(chalk.red('Missing enclosure :('));
}
}
}
} else {
logger.info(chalk.red("Didn't find any articles or episodes."));
}
let schema = feedType === 'rss' ? RSS : Podcast;
let lookup = { $or: [{ feedUrl: target }, { feedUrl: normalize(target) }] };
if (program.task) {
logger.info('trying to create a task on the bull queue');
let instance = await schema.findOne(lookup);
schema
.findOne(lookup)
.catch((err) => {
console.log('failed', err);
})
.then((instance) => {
let queuePromise;
if (!instance) {
logger.info('failed to find publication');
return;
}
if (feedType == 'rss') {
queuePromise = RssQueueAdd(
{
rss: instance._id,
url: instance.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
} else {
queuePromise = PodcastQueueAdd(
{
podcast: instance._id,
url: instance.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
}
queuePromise
.then(() => {
if (feedType === 'rss') {
logger.info(
`Scheduled RSS feed to the queue for parsing ${target} with id ${instance._id}`,
);
} else {
logger.info(
`Scheduled Podcast to the queue for parsing ${target}`,
);
}
})
.catch((err) => {
logger.error('Failed to schedule task on og queue');
});
});
}
}
if (feedType === 'rss') {
let feedContent = await ParseFeed(target, 2);
validate(feedContent);
} else {
let podcastContent = await ParsePodcast(target, 2);
validate(podcastContent);
}
logger.info('Note that upgrading feedparser can sometimes improve parsing.');
}
}
================================================
FILE: api/src/commands/cleanup-follows.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
import Follow from '../models/follow';
import { FollowSchema } from '../models/follow';
import RSS from '../models/rss';
import User from '../models/user';
import asyncTasks from '../asyncTasks';
program.parse(process.argv);
async function main() {
logger.info(`time to update those follow counts, \\0/`);
// get the follow counts
let counts = await Follow.aggregate([
{
$group: {
_id: {
rss: '$rss',
podcast: '$podcast',
user: '$user',
},
count: { $sum: 1 },
},
},
{
$match: {
count: { $gte: 2 },
},
},
]);
// group by the number of followers and the type for fast updates
let grouped = {};
for (let c of counts) {
let lookup = c._id;
let debug = JSON.stringify(lookup);
if (Object.keys(lookup).length != 2) {
throw Error(`OH no you dont, broken lookup: ${debug}`);
}
let versions = await Follow.find(lookup);
// remove everything except the first result
if (versions.length > 1) {
let removing = versions.length - 1;
logger.info(`removing ${removing} instances for query ${debug}`);
for (let v of versions.slice(1)) {
logger.info(`removing ${v._id}`);
await v.remove();
}
}
}
logger.info(`finished cleaning up, applying unqiue index now`);
await FollowSchema.index({ user: 1, rss: 1, podcast: 1 }, { unique: true });
logger.info(`done :)`);
}
main()
.then((result) => {
logger.info('completed it all, we should now have a unique contraint');
})
.catch((err) => {
logger.info(`failed with err ${err}`, { err });
});
================================================
FILE: api/src/commands/denormalize-follows.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
import Follow from '../models/follow';
import RSS from '../models/rss';
import asyncTasks from '../asyncTasks';
program.parse(process.argv);
async function main() {
logger.info(`time to update those follow counts, \\0/`);
// get the follow counts
let counts = await Follow.aggregate([
{
$group: {
_id: {
rss: '$rss',
podcast: '$podcast',
},
followers: { $sum: 1 },
},
},
]);
// group by the number of followers and the type for fast updates
let grouped = {};
for (let c of counts) {
c.type = c._id.rss ? 'rss' : 'podcast';
c.publicationID = c._id[c.type];
let key = [c.followers, c.type];
if (key in grouped) {
grouped[key].push(c);
} else {
grouped[key] = [c];
}
}
// update time
for (let group of Object.values(grouped)) {
let publicationIDs = group.map((c) => {
return c.publicationID;
});
let schema = group[0].type == 'rss' ? RSS : Podcast;
let followerCount = group[0].followers;
logger.info(
`Starting update of ${publicationIDs.length} publications of type ${group[0].type} to count ${followerCount}`,
);
let result = await schema.update(
{
$and: [
{ _id: { $in: publicationIDs } },
{ followerCount: { $ne: followerCount } },
],
},
{
followerCount: followerCount,
},
{ multi: true },
);
logger.info(`Updated ${result.nModified} out of ${publicationIDs.length}`);
}
// set everyone else to 0
for (let schema of [RSS, Podcast]) {
let result = await schema.update(
{ followerCount: { $exists: false } },
{
followerCount: 0,
},
{ multi: true },
);
logger.info(`Updated ${result.nModified} with no values to 0`);
}
}
main()
.then((result) => {
logger.info('completed it all, open the test page to see queue status');
})
.catch((err) => {
logger.info(`failed with err ${err}`, { err });
});
================================================
FILE: api/src/commands/denormalize-pins.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
import Follow from '../models/follow';
import RSS from '../models/rss';
import Pin from '../models/pin';
import User from '../models/user';
import asyncTasks from '../asyncTasks';
program.parse(process.argv);
async function main() {
logger.info(`time to denormalize those pin urls, \\0/`);
// denormalize the pin urls
let pins = await Pin.find({});
let counts = { denormalized: 0, normalized: 0, missing: 0, brokenref: 0 };
for (let p of pins) {
let url = (p.article && p.article.url) || (p.episode && p.episode.url);
if (!p.url && p.user) {
if (url) {
p.url = url;
await p.save();
counts.denormalized += 1;
} else {
if (p._id) {
await Pin.deleteOne({ _id: p._id });
}
counts.brokenref += 1;
}
}
logger.info(`pin url ${p.url} is now denormalized`);
}
// restore the relation for pins where we miss a matching schema
pins = await Pin.find({}).lean();
for (let p of pins) {
let type = p.article ? 'rss' : 'podcast';
let postType = type == 'rss' ? 'article' : 'episode';
let schema = type == 'rss' ? Article : Episode;
let postID = p.article || p.episode;
let instance = await schema.findOne({ _id: postID });
if (!instance && p.url) {
logger.info(`restoring relation for pin with url ${p.url}`);
let newInstance = await schema.findOne({ url: p.url }).lean();
if (newInstance) {
let data = {};
data[postType] = newInstance._id;
let result = await Pin.updateOne({ _id: p._id }, data);
logger.info(
`found a new instance for ${p._id}: ${p.url} with id ${newInstance._id}`,
);
counts.normalized += 1;
} else {
if (p._id) {
await Pin.deleteOne({ _id: p._id });
}
logger.info(`couldnt find a new instance :(`);
counts.missing += 1;
}
}
}
let c = JSON.stringify(counts);
logger.info(`completed, counts are ${c}`);
}
main()
.then((result) => {
logger.info('completed it all, open the test page to see queue status');
})
.catch((err) => {
logger.info(`failed with err ${err}`, { err });
});
================================================
FILE: api/src/commands/email.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
import Follow from '../models/follow';
import RSS from '../models/rss';
import Pin from '../models/pin';
import User from '../models/user';
import { SendWeeklyEmail, CreateWeeklyEmail } from '../utils/email/send';
import { weeklyContextGlobal, weeklyContextUser } from '../utils/email/context';
import asyncTasks from '../asyncTasks';
import config from '../config';
import jwt from 'jsonwebtoken';
import axios from 'axios';
import * as personalization from '../utils/personalization';
program.option('--send', 'Actually send the email').parse(process.argv);
async function main() {
logger.info(`time to send article recommendations, \\0/`);
// prep the data we need for everyone
let globalContext = await weeklyContextGlobal();
let users = await User.find({});
let enabledUsers = users.filter((u) => {
return u.weeklyEmail || u.admin;
});
logger.info(`going to email ${enabledUsers.length} users`);
for (const u of enabledUsers) {
let userContext = await weeklyContextUser(u);
let context = Object.assign({}, userContext, globalContext);
let obj = CreateWeeklyEmail(context);
logger.info(`email ${obj.html}`);
if (program.send) {
logger.info(`sending email, yee to user ${u.email}`);
SendWeeklyEmail(context);
}
}
}
main()
.then((result) => {
logger.info('all done sending emails');
})
.catch((err) => {
logger.info(`failed with err ${err}`, { err });
});
================================================
FILE: api/src/commands/load-featured-feeds.js
================================================
import '../loadenv';
import program from 'commander';
import logger from '../utils/logger';
import fs from 'fs';
import async from 'async';
import rssFinder from 'rss-finder';
import normalizeUrl from 'normalize-url';
import podcastFinder from 'rss-finder';
import { ParsePodcast } from '../parsers';
import strip from 'strip';
import Podcast from '../models/podcast';
import RSS from '../models/rss';
import moment from 'moment';
import '../utils/db';
import entities from 'entities';
import path from 'path';
import search from '../utils/search';
import asyncTasks from '../asyncTasks';
const version = '0.0.1';
program
.version(version)
.option('--type <value>', 'The type: episode, podcast or article')
.option('--url <value>', 'The url to try and scrape')
.option('--task', 'Create a task on bull or not')
.parse(process.argv);
process.on('unhandledRejection', (err) => {
console.error(err);
process.exit(1);
});
// TODO: refactor all code used in this command
function main() {
// This is a small helper tool to quickly help debug issues with podcasts or RSS feeds
logger.info('Starting to load the featured feeds');
var featured = JSON.parse(
fs.readFileSync(path.join('..', 'fixtures', 'featured.json'), 'utf8'),
);
async.mapLimit(
featured.rss,
10,
(featuredRSS, loopCb) => {
logger.info(`Now Handling RSS Feed ${featuredRSS.name}`);
rssFinder(normalizeUrl(featuredRSS.feedUrl))
.catch(function (err) {
logger.warn(
`RSS Finder broke ${featuredRSS.feedUrl} with err ${err}`,
);
loopCb();
return;
})
.then((feeds) => {
if (!feeds.feedUrls.length) {
logger.warn(
`We couldn't find any feeds for that RSS feed URL :( ${featuredRSS.feedUrl}`,
);
return loopCb();
}
async.mapLimit(
feeds.feedUrls,
feeds.feedUrls.length,
(feed, cb) => {
let feedTitle = feed.title;
if (feedTitle.toLowerCase() === 'rss') {
feedTitle = feeds.site.title;
}
var promise = RSS.findOneAndUpdate(
{ feedUrl: feed.url },
{
interest: featuredRSS.category,
categories: 'RSS',
description: (
entities.decodeHTML(feed.title) || ''
).substring(0, 240),
featured: false,
feedUrl: feed.url,
images: {
favicon: feeds.site.favicon,
},
lastScraped: moment().format(),
title: featuredRSS.name,
url: feeds.site.url,
valid: true,
},
{
new: true,
rawResult: true,
upsert: true,
},
);
promise
.then((rss) => {
if (rss.lastErrorObject.updatedExisting) {
cb(null, rss.value);
} else {
search(rss.value.searchDocument())
.then(() => {
return asyncTasks.RssQueueAdd(
{
rss: rss.value._id,
url: rss.value.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
})
.then(() => {
cb(null, rss.value);
})
.catch((err) => {
cb(err);
});
}
})
.catch((err) => {
logger.warn('broken', err);
cb(err);
});
},
(err, results) => {
if (err) {
logger.warn('really broken', err);
return loopCb();
}
loopCb();
},
);
});
},
function () {
logger.info('Finished with Feeds');
},
);
async.mapLimit(
featured.podcasts,
10,
(featuredPodcast, loopCb) => {
logger.info(`Now Handling Podcast ${featuredPodcast.name}`);
podcastFinder(normalizeUrl(featuredPodcast.feedUrl))
.catch((err) => {
logger.error(`podcastFinder broke ${featuredPodcast.feedUrl}`);
loopCb();
})
.then((feeds) => {
if (!feeds || !feeds.feedUrls.length) {
logger.error(`no feeds found for ${featuredPodcast.feedUrl}`);
return loopCb();
}
async.mapLimit(
feeds.feedUrls,
feeds.feedUrls.length,
(feed, cb) => {
// Get more metadata
ParsePodcast(feed.url, function (err, podcastContents) {
let title, url, images, description;
if (podcastContents) {
title =
strip(podcastContents.title) || strip(feed.title);
url = podcastContents.link || feeds.site.url;
images = {
favicon: feeds.site.favicon,
og: podcastContents.image,
};
description = podcastContents.description;
} else {
title = strip(feed.title);
url = feeds.site.url;
images = { favicon: feeds.site.favicon };
description = '';
}
Podcast.findOneAndUpdate(
{ feedUrl: feed.url },
{
interest: featuredPodcast.category,
categories: 'podcast',
description: (description || '').substring(
0,
240,
),
featured: false,
feedUrl: feed.url,
images: images,
lastScraped: new Date(0),
title: featuredPodcast.name,
url: normalizeUrl(url),
valid: true,
},
{
new: true,
rawResult: true,
upsert: true,
},
)
.then((podcast) => {
if (podcast.lastErrorObject.updatedExisting) {
cb(null, podcast.value);
} else {
search(podcast.value.searchDocument())
.then(() => {
return asyncTasks.PodcastQueueAdd(
{
podcast: podcast.value._id,
url: podcast.value.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
})
.then(() => {
logger.info(
`api is scheduling ${podcast.value.url} for og scraping`,
);
if (!podcast.value.images.og) {
asyncTasks
.OgQueueAdd(
{
url:
podcast.value.url,
type: 'podcast',
},
{
removeOnComplete: true,
removeOnFail: true,
},
)
.then(() => {
cb(null, podcast.value);
});
} else {
cb(null, podcast.value);
}
})
.catch((err) => {
cb(err);
});
}
})
.catch((err) => {
logger.error(
`Podcast parsing broke for ${feed.url}`,
);
cb(err);
});
});
},
(err, results) => {
if (err) {
logger.error(`broken stuff with error ${err}`);
loopCb();
return;
}
loopCb();
},
);
});
},
function () {
logger.info('finished with podcasts');
},
);
}
main();
================================================
FILE: api/src/commands/rescrape-favicon.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import RSS from '../models/rss';
import { discoverRSS } from '../parsers/discovery';
import { isURL } from '../utils/validation';
program
.option('--all', 'Rescrape articles for which we already have a favicon image')
.option('-c, --concurrency <n>', 'The number of concurrent scraping updates', 100)
.option('--publication <s>', 'ID of the publication to scrape')
.parse(process.argv);
async function rescrapeFavicon(publicationType, instance) {
const schema = publicationType == 'rss' ? RSS : Podcast;
if (!isURL(instance.url)) {
return;
}
try {
let foundRSS = await discoverRSS(instance.url);
if (foundRSS && foundRSS.site && foundRSS.site.favicon) {
let site = foundRSS.site;
const images = instance.images || {};
let updated;
if (images.favicon != site.favicon) {
images.favicon = site.favicon;
updated = await schema.update({ _id: instance._id }, { images });
}
logger.info(`updated ${instance._id} to url ${site.favicon}`);
return updated;
}
} catch (err) {
logger.warn(
`rescraping failed with error for url ${instance.url} with instance id ${instance._id}`,
);
}
}
async function main() {
let schemas = { rss: RSS, podcast: Podcast };
logger.info(`program.all is set to ${program.all}`);
let counts = { hasimage: 0, fixed: 0, notfound: 0 };
let lookup = { url: { $nin: [null, ''] } };
if (program.publication) {
lookup['_id'] = program.publication;
} else if (!program.all) {
lookup['images.favicon'] = { $in: [null, ''] };
}
let chunkSize = parseInt(program.concurrency, 10);
for (const [contentType, schema] of Object.entries(schemas)) {
let total = await schema.count(lookup);
let completed = 0;
logger.info(
`Found ${total} for ${contentType}, processing in chunks of ${chunkSize}`,
);
for (let i = 0, j = total; i < j; i += chunkSize) {
let chunk = await schema
.find(lookup)
.sort({ followerCount: -1 })
.skip(i)
.limit(chunkSize)
.lean();
completed = completed + chunkSize;
let promises = [];
for (const instance of chunk) {
let missingImage = !instance.images || !instance.images.favicon;
if (missingImage || program.all) {
let promise = rescrapeFavicon(contentType, instance);
promises.push(promise);
} else {
counts.hasimage += 1;
}
}
let results = await Promise.all(promises);
for (const r of results) {
if (r) {
counts.fixed += 1;
} else {
counts.notfound += 1;
}
}
console.log('completed', completed);
}
console.log('counts', counts);
logger.info(`Completed for type ${contentType}`);
}
}
main()
.then((result) => {
logger.info('completed it all');
})
.catch((err) => {
logger.warn(`failed with err ${err}`);
});
================================================
FILE: api/src/commands/rescrape-og.js
================================================
import '../utils/db';
import program from 'commander';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
import RSS from '../models/rss';
import { OgQueueAdd } from '../asyncTasks';
program
.option('--all', 'Rescrape articles for which we already have an og image')
.parse(process.argv);
function partitionBy(collection, selector) {
if (!collection.length) {
return [];
}
const partitions = [[collection[0]]];
let currentPartition = 0;
let lastElement = selector(collection[0]);
for (let i = 1; i < collection.length; ++i) {
const element = selector(collection[i]);
if (element !== lastElement) {
partitions.push([]);
++currentPartition;
}
partitions[currentPartition].push(collection[i]);
lastElement = element;
}
return partitions;
}
async function main() {
const schemas = { podcast: Podcast, rss: RSS, episode: Episode, article: Article };
const fieldMap = { article: 'url', episode: 'link', podcast: 'url', rss: 'url' };
const feedIdMap = { episode: 'podcast', article: 'rss', rss: '_id', podcast: '_id' };
const feedFieldMap = {
episode: 'podcast',
article: 'rss',
rss: 'rss',
podcast: 'podcast',
};
console.log(`program.all is set to ${program.all}`);
for (const [contentType, schema] of Object.entries(schemas)) {
const total = await schema.count({});
const chunkSize = 1000;
const field = fieldMap[contentType];
const feedField = feedFieldMap[contentType];
const feedIdField = feedIdMap[contentType];
console.log(`Found ${total} for ${contentType} with url field ${field}\n`);
let completed = 0;
for (let i = 0, j = total; i < j; i += chunkSize) {
const chunk = await schema
.find()
.sort(feedIdField)
.skip(i)
.limit(chunkSize)
.lean();
completed += chunkSize;
const instances = chunk.filter((instance) => {
const missingImage = !instance.images || !instance.images.og;
return (missingImage || program.all) && instance[field];
});
const partitions = partitionBy(instances, (i) => i[feedIdField]);
const promises = partitions.map((partition) => {
return OgQueueAdd(
{
type: contentType,
[feedField]: partition[0][feedIdField],
urls: partition.map((i) => i[field]),
update: true,
},
{ removeOnComplete: true, removeOnFail: true },
);
});
await Promise.all(promises);
const progress = Math.floor((100 * i) / j);
console.log(`\rprogress ${progress}%: ${i}/${j}`);
}
console.log(`Completed for type ${contentType} with field ${field}`);
}
}
main()
.then((result) => {
console.log('completed it all, open the test page to see queue status');
process.exit(0);
})
.catch((err) => {
console.log(`failed: ${err.stack}`);
process.exit(1);
});
================================================
FILE: api/src/commands/reset-parsing-state.js
================================================
import '../loadenv';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import '../utils/db';
import logger from '../utils/logger';
logger.info('Starting the RSS reset');
// simple script to reset isParsing state on Podcast and RSS feeds
async function main() {
logger.info('Updating RSS feeds now');
let rssResponse = await RSS.update(
{},
{ 'queueState.isParsing': false },
{ multi: true },
);
logger.info(`Updated isParsing to false for ${rssResponse.nModified} RSS feeds`);
logger.info('Updating Podcast feeds now');
let podcastResponse = await Podcast.update(
{},
{ 'queueState.isParsing': false },
{ multi: true },
);
logger.info(
`Updated isParsing to false for ${podcastResponse.nModified} Podcast feeds`,
);
}
main()
.then(() => {
logger.info('Finished reset for podcast and rss feeds');
})
.catch((err) => {
logger.error(`Something went wrong with the reset ${err}`);
});
================================================
FILE: api/src/commands/resync-follows.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import logger from '../utils/logger';
import Follow from '../models/follow';
import { FollowSchema } from '../models/follow';
import asyncTasks from '../asyncTasks';
import stream from 'getstream';
import config from '../config';
const streamClient = stream.connect(config.stream.apiKey, config.stream.apiSecret);
program.parse(process.argv);
async function main() {
logger.info(`time to resync those follows, \\0/`);
let followCount = await Follow.count({});
let chunkSize = 500;
for (let i = 0, j = followCount; i < j; i += chunkSize) {
let follows = await Follow.find({}).skip(i).limit(chunkSize).lean();
logger.info(`found ${follows.length} follows`);
let feedRelations = [];
for (let f of follows) {
let feedGroup = f.rss ? 'user_article' : 'user_episode';
let type = f.rss ? 'rss' : 'podcast';
let publicationID = f.rss || f.podcast;
// sync to stream
feedRelations.push({
source: `timeline:${f.user}`,
target: `${type}:${publicationID}`,
});
feedRelations.push({
source: `${feedGroup}:${f.user}`,
target: `${type}:${publicationID}`,
});
}
logger.info(`pushed ${feedRelations.length} follows to Stream`);
logger.info(`completed ${i} out of ${followCount}`);
let response = await streamClient.followMany(feedRelations);
}
logger.info(`completed all loops`);
}
main()
.then((result) => {
logger.info('completed it all, we should now have a unique contraint');
})
.catch((err) => {
logger.info(`failed with err ${err}`, { err });
});
================================================
FILE: api/src/commands/winds-article.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import chalk from 'chalk';
import logger from '../utils/logger';
const version = '0.0.1';
import normalize from 'normalize-url';
import { ParseArticle } from '../parsers/article';
program.version(version).parse(process.argv);
let articleUrls = program.args;
async function main() {
logger.info('Starting the article parsing debugger \\0/');
for (let url of articleUrls) {
let scraped = await ParseArticle(url);
logger.info(Object.keys(scraped.data));
logger.info(`excerpt: ${scraped.data.excerpt}`);
}
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-discover.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import chalk from 'chalk';
import logger from '../utils/logger';
const version = '0.0.1';
import normalize from 'normalize-url';
import asyncTasks from '../asyncTasks';
import { ParseArticle } from '../parsers/article';
import { discoverRSS } from '../parsers/discovery';
program.version(version).parse(process.argv);
let pageUrls = program.args;
async function main() {
logger.info('Starting the article parsing debugger \\0/');
for (let url of pageUrls) {
let foundRSS = await discoverRSS(normalize(url));
if (!foundRSS.feedUrls.length) {
logger.info('no RSS found');
return;
}
let site = foundRSS.site;
logger.info(`Site Information`);
logger.info(`Title: ${site.title}, URL: ${site.url}, Favicon: ${site.favicon}`);
logger.info(`Favicon ${foundRSS.site.favicon}`);
logger.info(`RSS feeds found: ${foundRSS.feedUrls.length}`);
for (let found of foundRSS.feedUrls) {
logger.info(`Title: ${found.title} URL: ${found.url}`);
}
}
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-merge.js
================================================
import mongoose from 'mongoose';
import ProgressBar from 'progress';
import db from '../utils/db';
import { upsertCollections } from '../utils/collections';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
function sleep(time) {
return new Promise((resolve) => setTimeout(resolve, time));
}
process.on('unhandledRejection', (error) => console.error(error.message));
const feedModels = {
rss: { feed: RSS, content: Article },
podcast: { feed: Podcast, content: Episode },
};
function estimateSize(content) {
let size = 2; // {}
for (const [key, value] of Object.entries(content)) {
size += Buffer.byteLength(String(key), 'utf8');
size += Buffer.byteLength(String(value), 'utf8');
size += 2; // :,
}
return size;
}
async function main() {
await db;
for (const type of ['rss', 'podcast']) {
const model = feedModels[type];
console.log(`synchronising ${type} content`);
const contentModelName = model.content.collection.collectionName;
const feedCount = await model.feed.countDocuments();
const bar = new ProgressBar(
'[:current / :total] :bar [:percent | :rate records per second]',
{ total: feedCount },
);
let lastFeedId = mongoose.Types.ObjectId('000000000000000000000000');
let feedsSynced = false;
while (!feedsSynced) {
try {
const feedCursor = model.feed.collection
.find({
_id: { $gte: lastFeedId },
})
.sort({
_id: 1,
})
.batchSize(32);
while (await feedCursor.hasNext()) {
const feed = await feedCursor.next();
const allowedLanguage = [null, undefined, '', 'eng'].includes(
feed.language,
);
if (!allowedLanguage) {
bar.tick();
continue;
}
let lastId = mongoose.Types.ObjectId('000000000000000000000000');
let articlesSynced = false;
while (!articlesSynced) {
try {
const cursor = model.content.collection
.find({
_id: { $gte: lastId },
[type]: feed._id,
})
.sort({
publicationDate: 1,
_id: 1,
})
.limit(1000)
.batchSize(1000);
const articleCount = await model.content.countDocuments({
[type]: feed._id,
});
let upserts = [];
let content = [];
let currentSize = 0;
let mostRecentPublicationDate;
const chunkSize = 1000;
const sizeLimit = 100 * 1024; // less then 128Kb to leave some space for external data
while (await cursor.hasNext()) {
const source = await cursor.next();
mostRecentPublicationDate = source.publicationDate;
const item = {
id: source._id,
title: source.title,
likes: source.likes,
socialScore: source.socialScore,
description: (source.description || '').substring(
0,
240,
),
publicationDate: source.publicationDate,
[type]: source[type],
};
//XXX: we overestimate object size by 5-10%
const size = estimateSize(item);
const batchIsFull = content.length == chunkSize;
const batchTooBig = currentSize + size > sizeLimit;
if (batchIsFull || batchTooBig) {
upserts.push(
upsertCollections(contentModelName, content),
);
content = [];
currentSize = 0;
}
if (upserts.length && upserts.length % 128 === 0) {
await Promise.all([
...upserts,
// sleep(800)
]);
upserts = [];
}
currentSize += size;
content.push(item);
lastId = item.id;
}
await Promise.all([
...upserts,
upsertCollections(contentModelName, content),
upsertCollections(type, [
{
id: feed._id,
title: feed.title,
language: feed.language,
description: (feed.description || '').substring(
0,
240,
),
articleCount: content.length,
mostRecentPublicationDate,
},
]),
// sleep(800)
]);
bar.tick();
articlesSynced = true;
} catch (err) {
console.error(
`\n\terror processing content: ${err.message}\nresuming from ${lastId}`,
);
}
}
lastFeedId = feed._id;
}
console.log(`${type} synchronised\n`);
feedsSynced = true;
} catch (err) {
console.error(
`\n\terror processing feed: ${err.message}\nresuming from ${lastFeedId}`,
);
}
}
}
}
main()
.then(() => {
console.log('\ndone');
process.exit(0);
})
.catch((err) => {
console.error(`\nfailed with err ${err.stack}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-og.js
================================================
import '../loadenv';
import '../utils/db';
import program from 'commander';
import chalk from 'chalk';
import logger from '../utils/logger';
const version = '0.0.1';
import normalize from 'normalize-url';
import { OgQueueAdd } from '../asyncTasks';
import { ParseOG, IsValidOGUrl } from '../parsers/og';
program
.version(version)
.option('--type <value>', 'The type: episode, podcast or article')
.option('--task', 'Create a task on bull or not')
.parse(process.argv);
let ogUrls = program.args;
async function main() {
// This is a small helper tool to quickly help debug issues with podcasts or RSS feeds
logger.info('Starting the OG queue Debugger \\0/');
for (let ogUrl of ogUrls) {
let isValid = await IsValidOGUrl(ogUrl);
if (!isValid) {
logger.warn(`invalid URL ${ogUrl}`);
continue;
}
let normalizedUrl = normalize(ogUrl);
if (!normalizedUrl) {
logger.warn(`no normalized URL for '${ogUrl}'`);
continue;
}
logger.info(`Looking for og images at ${normalizedUrl} for type ${program.type}`);
let ogImage = await ParseOG(normalizedUrl);
if (!ogImage) {
logger.info(
chalk.red(`OG scraping didn't find an image for ${normalizedUrl}`),
);
} else {
logger.info(chalk.green(`Image found for ${normalizedUrl}: ${ogImage}`));
}
if (program.task) {
logger.info('creating a task on the bull queue');
let res = await OgQueueAdd(
{
url: normalizedUrl,
type: program.type,
update: true,
},
{
removeOnComplete: true,
removeOnFail: true,
},
);
}
}
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-podcast.js
================================================
import program from 'commander';
import '../loadenv';
import '../utils/db';
import { debugFeed } from './_debug-feed';
program
.option('-t, --task', 'create a task')
.option('-l, --limit <n>', 'The number of articles to parse', 2)
.action((feedUrl, cmd) => {
debugFeed('podcast', [feedUrl]);
})
.parse(process.argv);
================================================
FILE: api/src/commands/winds-rebuild-search.js
================================================
import '../loadenv';
import '../utils/db';
import logger from '../utils/logger';
import Podcast from '../models/podcast';
import RSS from '../models/rss';
import { indexMany } from '../utils/search';
async function main() {
logger.info('Reindexing all Podcasts to Algolia');
await loadModel(Podcast);
logger.info('Reindexing all RSS feeds to Algolia');
await loadModel(RSS);
}
async function loadModel(Model) {
const batchSize = 200;
let indexed = 0;
let accumulatedDocs = [];
// XXX: enter Mongoose genius: { timeout: true } means disabling cursor timeouts
await Model.find({ followerCount: { $gte: 1 } }, {}, { timeout: true })
.cursor()
.eachAsync(async (d) => {
accumulatedDocs.push(d.searchDocument());
if (accumulatedDocs.length >= batchSize) {
await indexMany(accumulatedDocs);
indexed += accumulatedDocs.length;
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Indexed: ${indexed}`);
accumulatedDocs = [];
}
});
if (accumulatedDocs.length >= 0) {
await indexMany(accumulatedDocs);
indexed += accumulatedDocs.length;
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Indexed: ${indexed}`);
accumulatedDocs = [];
}
process.stdout.write('\n');
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-rehash.js
================================================
import '../loadenv';
import '../utils/db';
import logger from '../utils/logger';
import Article from '../models/article';
import RSS from '../models/rss';
async function main() {
logger.info('Re-hasing all articles');
await rehashModel(Article);
}
async function rehashModel(Model) {
let indexed = 0;
let promises = [];
let batchSize = 100;
await Model.find({}, { rss: 0 }, { timeout: true })
.cursor()
.eachAsync(async (d) => {
d.computeContentHash();
promises.push(d.save());
indexed += 1;
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Indexed: ${indexed}`);
if (promises.length > batchSize) {
await Promise.all(promises);
promises = [];
}
});
process.stdout.write('\n');
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds-rss.js
================================================
import program from 'commander';
import '../loadenv';
import '../utils/db';
import { debugFeed } from './_debug-feed';
program
.option('-t, --task', 'create a task')
.option('-l, --limit <n>', 'The number of articles to parse', 2)
.action((feedUrl, cmd) => {
debugFeed('rss', [feedUrl]);
})
.parse(process.argv);
================================================
FILE: api/src/commands/winds-truncate-rss-feed.js
================================================
import '../loadenv';
import '../utils/db';
import RSS from '../models/rss';
import Article from '../models/article';
import program from 'commander';
program.parse(process.argv);
let args = program.args;
async function main() {
let rss = await RSS.findById(args[0]).exec();
await Article.deleteMany({ rss: rss._id }, {}, { timeout: true }).exec();
}
main()
.then(() => {
console.info('done');
process.exit(0);
})
.catch((err) => {
console.info(`failed with err ${err}`);
process.exit(1);
});
================================================
FILE: api/src/commands/winds.js
================================================
#!/usr/bin/env babel-node
import program from 'commander';
import logger from '../utils/logger';
let version;
if (process.env.DOCKER) {
version = { version: 'DOCKER' };
} else {
version = require('../../../app/package.json');
}
program
.version(version)
.command('og <urls>', 'Debug OG')
.command('rss', 'Debug RSS feeds')
.command('podcast', 'Debug Podcasts')
.command('article', 'Debug Article Parsing')
.command('discover', 'Debug RSS discovery')
.command('rebuild-search', 'Rebuild search')
.command('rehash', 'Rehash articles and episodes')
.command('truncate-rss-feed <id>', 'Truncate articles for one RSS feed')
.parse(process.argv);
function main() {
logger.info('Winds CLI, Have fun!');
}
main();
================================================
FILE: api/src/config/dev.js
================================================
module.exports = {
url: 'https://winds.getstream.io',
logger: { level: process.env.LOGGER_LEVEL || 'info' },
email: {
backend: 'not-sendgrid',
sender: { support: { email: 'not.a.real.email@getstream.io' } },
},
};
================================================
FILE: api/src/config/index.js
================================================
import dotenv from 'dotenv';
import path from 'path';
const configs = {
development: { config: 'dev' },
production: { config: 'prod' },
test: {
config: 'test',
env: path.resolve(__dirname, '..', '..', 'test', '.env'),
},
};
const currentEnvironment = process.env.NODE_ENV || 'development';
const defaultPath = path.resolve(__dirname, '..', '..', '..', 'app', '.env');
const envPath = configs[currentEnvironment].env || defaultPath;
console.log(`Loading .env from '${envPath}'`);
dotenv.config({ path: envPath });
const _default = {
product: {
url: process.env.PRODUCT_URL,
name: process.env.PRODUCT_NAME,
author: process.env.PRODUCT_AUTHOR,
},
server: {
port: process.env.API_PORT,
},
jwt: {
secret: process.env.JWT_SECRET,
},
database: {
uri: process.env.DATABASE_URI,
},
cache: {
uri: process.env.CACHE_URI,
},
algolia: {
appId: process.env.REACT_APP_ALGOLIA_APP_ID,
writeKey: process.env.ALGOLIA_WRITE_KEY,
index: process.env.ALGOLIA_INDEX,
},
logger: {
level: process.env.LOGGER_LEVEL || 'warn',
host: process.env.LOGGER_HOST,
port: process.env.LOGGER_PORT,
},
sentry: {
dsn: process.env.SENTRY_DSN,
},
url: process.env.BASE_URL,
email: {
backend: 'sendgrid',
sender: {
default: {
name: process.env.EMAIL_SENDER_DEFAULT_NAME,
email: process.env.EMAIL_SENDER_DEFAULT_EMAIL,
},
support: {
name: process.env.EMAIL_SENDER_SUPPORT_NAME,
email: process.env.EMAIL_SENDER_SUPPORT_EMAIL,
},
},
sendgrid: {
secret: process.env.EMAIL_SENDGRID_SECRET,
},
},
stream: {
appId: process.env.STREAM_APP_ID,
apiKey: process.env.STREAM_API_KEY,
apiSecret: process.env.STREAM_API_SECRET,
baseUrl: process.env.STREAM_API_BASE_URL,
pro: process.env.STREAM_PLAN === 'pro',
},
analyticsDisabled: process.env.ANALYTICS_DISABLED || false,
statsd: {
host: process.env.STATSD_HOST || 'localhost',
port: process.env.STATSD_PORT || 8125,
prefix: process.env.STATSD_PREFIX || '',
},
newrelic: false,
social: {
reddit: {
username: process.env.REDDIT_USERNAME,
password: process.env.REDDIT_PASSWORD,
key: process.env.REDDIT_APP_ID,
secret: process.env.REDDIT_APP_SECRET,
},
},
};
const config = require(`./${configs[currentEnvironment].config}`);
module.exports = Object.assign({ env: currentEnvironment }, _default, config);
================================================
FILE: api/src/config/prod.js
================================================
module.exports = {};
================================================
FILE: api/src/config/test.js
================================================
module.exports = {
database: {
uri: 'mongodb://localhost:27017/test',
},
cache: {
uri: 'redis://localhost:6379/10',
},
email: {
backend: 'dummy',
sender: {
default: {
name: process.env.EMAIL_SENDER_DEFAULT_NAME,
email: process.env.EMAIL_SENDER_DEFAULT_EMAIL,
},
support: {
name: process.env.EMAIL_SENDER_SUPPORT_NAME,
email: process.env.EMAIL_SENDER_SUPPORT_EMAIL,
},
},
sendgrid: {
secret: process.env.EMAIL_SENDGRID_SECRET,
},
},
analyticsDisabled: true,
url: 'https://winds.gestream.io',
};
================================================
FILE: api/src/controllers/alias.js
================================================
import mongoose from 'mongoose';
import Alias from '../models/alias';
import Rss from '../models/rss';
import Podcast from '../models/podcast';
exports.list = async (req, res) => {
const query = req.query || {};
if (query.type === 'rss' || query.type === 'podcast') {
let obj = {};
obj[query.type] = { $exists: true };
obj['user'] = req.user.sub;
return res.json(await Alias.find(obj).sort({ _id: -1 }));
}
res.json(await Alias.find({ user: req.user.sub }));
};
exports.get = async (req, res) => {
const aliasId = req.params.aliasId;
if (!mongoose.Types.ObjectId.isValid(aliasId))
return res.status(400).json({
error: `Resource aliasId (${aliasId}) is an invalid ObjectId.`,
});
const alias = await Alias.findOne({ _id: aliasId, user: req.user.sub });
if (!alias) return res.status(404).json({ error: 'Resource does not exist.' });
res.json(alias);
};
exports.post = async (req, res) => {
const data = Object.assign({}, req.body, { user: req.user.sub });
const isRss = data.hasOwnProperty('rss');
const isPodcast = data.hasOwnProperty('podcast');
if (!(isRss || isPodcast)) return res.status(422).json({ error: 'Invalid request.' });
if (isRss && isPodcast) return res.status(422).json({ error: 'Invalid request.' });
if (!data.hasOwnProperty('alias'))
return res.status(422).json({ error: 'Missing required fields.' });
const exists = isRss
? await Rss.findById(data.rss)
: await Podcast.findById(data.podcast);
if (!exists) return res.status(422).json({ error: "Resource doesn't exists." });
const feedID = isRss ? { rss: data.rss } : { podcast: data.podcast };
if (!!(await Alias.findOne({ user: data.user, ...feedID }))) {
res.json(
await Alias.findOneAndUpdate(
{ user: data.user, ...feedID },
{ alias: data.alias },
{ new: true },
),
);
} else {
const alias = await Alias.create(data);
res.json(await Alias.findOne({ _id: alias._id }));
}
};
exports.put = async (req, res) => {
const aliasId = req.params.aliasId;
const newAlias = req.body.alias;
if (!newAlias) return res.status(422).json({ error: 'Missing required fields.' });
if (!mongoose.Types.ObjectId.isValid(aliasId))
return res.status(400).json({
error: `Resource aliasId (${aliasId}) is an invalid ObjectId.`,
});
const alias = await Alias.findById(aliasId);
if (!alias) return res.status(404).json({ error: 'Resource does not exist.' });
if (alias.user._id != req.User.id) return res.sendStatus(403);
res.json(await Alias.findByIdAndUpdate(aliasId, { alias: newAlias }, { new: true }));
};
exports.delete = async (req, res) => {
const alias = await Alias.findOne({ _id: req.params.aliasId, user: req.user.sub });
if (!alias) return res.status(404).json({ error: 'Resource does not exist.' });
await alias.remove();
res.sendStatus(204);
};
================================================
FILE: api/src/controllers/article.js
================================================
import mongoose from 'mongoose';
import Article from '../models/article';
import { getArticleRecommendations } from '../utils/personalization';
import { trackEngagement } from '../utils/analytics';
exports.list = async (req, res) => {
const query = req.query || {};
let articles = [];
if (query.type === 'recommended') {
articles = await getArticleRecommendations(req.User._id.toString());
} else {
if (query.rss && !mongoose.Types.ObjectId.isValid(query.rss)) {
return res.status(400).json({ error: `Invalid RSS id ${query.rss}` });
}
articles = await Article.apiQuery(req.query);
}
res.json(articles.filter((a) => a.valid));
};
exports.get = async (req, res) => {
let articleId = req.params.articleId;
if (!mongoose.Types.ObjectId.isValid(articleId)) {
return res
.status(400)
.json({ error: `Resource articleId (${articleId}) is an invalid ObjectId.` });
}
let article = await Article.findById(articleId);
if (!article) {
return res.status(404).json({ error: 'Resource not found.' });
}
if (req.query && req.query.type === 'parsed') {
let parsed;
try {
parsed = await article.getParsedArticle();
} catch (err) {
return res.status(400).json({ error: 'Failed to parse the article.' });
}
if (!parsed) {
return res.status(400).json({ error: 'Failed to parse the article.' });
}
await trackEngagement(req.User, {
label: 'open_article',
content: { foreign_id: `article:${articleId}` },
});
return res.json(parsed);
}
res.json(article);
};
================================================
FILE: api/src/controllers/auth.js
================================================
import { v4 as uuidv4 } from 'uuid';
import validator from 'validator';
import User from '../models/user';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import Follow from '../models/follow';
import config from '../config';
let packageInfo;
if (process.env.DOCKER) {
packageInfo = { version: 'DOCKER' };
} else {
packageInfo = require('../../../app/package.json');
}
import Redis from 'ioredis';
const cache = new Redis(config.cache.uri);
import { SendPasswordResetEmail, SendWelcomeEmail } from '../utils/email/send';
async function getInterestMap() {
const cacheKey = `interests:v${packageInfo.version.replace(/\./g, ':')}`;
let str = await cache.get(cacheKey);
let interestMap = JSON.parse(str);
if (!interestMap) {
interestMap = {};
const rss = await RSS.findFeatured();
const podcast = await Podcast.findFeatured();
for (let p of [...rss, ...podcast]) {
let k = p.interest || 'featured';
let d = p.toObject();
d.type = p.constructor.modelName == 'RSS' ? 'rss' : 'podcast';
if (!(k in interestMap)) {
interestMap[k] = [];
}
interestMap[k].push(d);
}
let cached = await cache.set(
cacheKey,
JSON.stringify(interestMap),
'EX',
60 * 30,
);
}
return interestMap;
}
exports.signup = async (req, res) => {
const data = Object.assign({}, { interests: [] }, req.body);
if (!data.name || !data.email || !data.username || !data.password) {
return res.status(400).json({ error: 'Missing required fields.' });
}
if (data.email && !validator.isEmail(data.email)) {
return res.status(400).json({ error: 'Invalid or malformed email address.' });
}
const regex = /^[\w-]+$/;
if (data.username && !regex.test(data.username)) {
return res.status(400).json({
error: 'Usernames must be alphanumeric but can only contain _, . or -.',
});
}
data.username = data.username.trim();
data.email = data.email.trim();
const exists = await User.findOne({
$or: [{ email: data.email }, { username: data.username }],
});
if (exists) {
return res.status(409).json({
error: 'A resource already exists with that username or email.',
});
}
const whitelist = Object.assign(
{},
...['name', 'email', 'username', 'password', 'interests'].map((key) => ({
[key]: data[key],
})),
);
const user = await User.create(whitelist);
let interestMap = await getInterestMap();
let interestFollow = interestMap['featured'] || [];
for (let i of data.interests) {
let publications = interestMap[i];
if (publications) {
interestFollow.push(...publications);
}
}
let followCommands = interestFollow.map((interest) => {
return {
type: interest.type,
publicationID: interest._id,
userID: user._id.toString(),
};
});
await Promise.all([
Follow.getOrCreateMany(followCommands),
SendWelcomeEmail({ email: user.email }),
]);
res.json(user.serializeAuthenticatedUser());
};
exports.login = async (req, res) => {
const data = req.body || {};
if (!data.email || !data.password) {
return res.status(400).json({ error: 'Missing required fields.' });
}
const email = data.email.toLowerCase().trim();
const user = await User.findOne({ email: email });
if (!user) {
return res.status(404).json({ error: 'Resource does not exist.' });
}
if (!(await user.verifyPassword(data.password))) {
return res.status(403).json({ error: 'Invalid username or password.' });
}
res.status(200).send(user.serializeAuthenticatedUser());
};
exports.forgotPassword = async (req, res) => {
const opts = { new: true };
const recoveryCode = uuidv4();
let email = req.body.email.toLowerCase();
const user = await User.findOneAndUpdate(
{ email: email },
{ recoveryCode: recoveryCode },
opts,
);
if (!user) {
return res.status(404).json({ error: 'Resource could not be found.' });
}
await SendPasswordResetEmail({ email: user.email, recoveryCode: user.recoveryCode });
res.sendStatus(200);
};
exports.resetPassword = async (req, res) => {
const user = await User.findOneAndUpdate(
{ email: req.body.email.toLowerCase(), recoveryCode: req.body.recoveryCode },
{ password: req.body.password },
{ new: true },
);
if (!user) {
return res.status(404).json({ error: 'Resource could not be found.' });
}
res.status(200).send(user.serializeAuthenticatedUser());
};
================================================
FILE: api/src/controllers/default.js
================================================
exports.get = (req, res) => {
res.status(200).send('pong');
};
exports.post = (req, res) => {
res.status(200).send('pong');
};
================================================
FILE: api/src/controllers/email.js
================================================
import mongoose from 'mongoose';
import User from '../models/user';
import {
dailyContextGlobal,
dailyContextUser,
weeklyContextGlobal,
weeklyContextUser,
} from '../utils/email/context';
import { CreateDailyEmail, CreateWeeklyEmail, SendEmail } from '../utils/email/send';
exports.list = async (req, res) => {
res.json(['daily', 'weekly']);
};
async function createEmail(type, user) {
const create = { daily: CreateDailyEmail, weekly: CreateWeeklyEmail };
const context = {
// storing as lambda to defer evaluation
daily: () => [dailyContextGlobal(), dailyContextUser(user)],
weekly: () => [weeklyContextGlobal(), weeklyContextUser(user)],
};
const emailContext = await Promise.all(context[type]());
return create[type](Object.assign({}, ...emailContext));
}
exports.get = async (req, res) => {
if (!['daily', 'weekly'].includes(req.params.emailName)) {
return res.status(401);
}
const userId = req.query.user;
if (!mongoose.Types.ObjectId.isValid(userId)) {
return res.status(400).json({ error: `Invalid user id ${userId}.` });
}
const user = await User.findOne({ _id: userId, admin: true });
if (!user) {
return res.status(404);
}
const email = await createEmail(req.params.emailName, user);
return res.type('html').send(email.html);
};
exports.post = async (req, res) => {
if (!['daily', 'weekly'].includes(req.params.emailName)) {
return res.status(401);
}
const userId = req.query.user;
if (!mongoose.Types.ObjectId.isValid(userId)) {
return res.status(400).json({ error: `Invalid user id ${userId}.` });
}
const user = await User.findOne({ _id: userId, admin: true });
if (!user) {
return res.status(404);
}
const email = await createEmail(req.params.emailName, user);
const result = await SendEmail(email);
return res.status(200).json(result);
};
================================================
FILE: api/src/controllers/episode.js
================================================
import Episode from '../models/episode';
import { getEpisodeRecommendations } from '../utils/personalization';
import { trackEngagement } from '../utils/analytics';
import mongoose from 'mongoose';
exports.list = async (req, res) => {
const query = req.query || {};
let episodes = [];
if (query.type === 'recommended') {
episodes = await getEpisodeRecommendations(req.User._id.toString());
} else {
if (query.podcast && !mongoose.Types.ObjectId.isValid(query.podcast)) {
return res.status(400).json({ error: `Invalid Podcast id ${query.podcast}` });
}
episodes = await Episode.apiQuery(req.query);
}
res.json(episodes);
};
exports.get = async (req, res) => {
const episodeId = req.params.episodeId;
if (episodeId === 'undefined') {
return res.status(404).json({ error: 'Missing required field episodeId.' });
}
if (!mongoose.Types.ObjectId.isValid(episodeId)) {
return res
.status(400)
.json({ error: `Resource episodeId (${episodeId}) is an invalid ObjectId.` });
}
let episode = await Episode.findById(episodeId);
if (!episode) {
return res.status(404).json({ error: 'Episode could not be found.' });
}
if (req.query && req.query.type === 'parsed') {
try {
const parsed = await episode.getParsedEpisode();
if (!parsed) {
return res.status(400).json({ error: 'Failed to parse the episode.' });
}
await trackEngagement(req.User, {
label: 'open_episode',
content: { foreign_id: `episode:${episodeId}` },
});
return res.json(parsed);
} catch (err) {
return res.status(400).json({ error: 'Failed to parse the episode.' });
}
}
res.json(episode);
};
================================================
FILE: api/src/controllers/featured.js
================================================
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import config from '../config';
let packageInfo;
if (process.env.DOCKER) {
packageInfo = { version: 'DOCKER' };
} else {
packageInfo = require('../../../app/package.json');
}
import Redis from 'ioredis';
const cache = new Redis(config.cache.uri);
exports.list = async (req, res) => {
const cacheKey = `featured:v${packageInfo.version.replace(/\./g, ':')}`;
let str = await cache.get(cacheKey);
let data = JSON.parse(str);
if (!data) {
data = [];
const rss = await RSS.find({ featured: true });
rss.map((doc) => {
let r = doc.toObject();
r.type = 'rss';
data.push(r);
});
const podcasts = await Podcast.find({ featured: true });
podcasts.map((doc) => {
let p = doc.toObject();
p.type = 'podcast';
data.push(p);
});
await cache.set(cacheKey, JSON.stringify(data), 'EX', 60 * 30);
}
let shuffled = data
.map((a) => [Math.random(), a])
.sort((a, b) => a[0] - b[0])
.map((a) => a[1]);
res.json(shuffled);
};
================================================
FILE: api/src/controllers/feed.js
================================================
import Article from '../models/article';
import Episode from '../models/episode';
import config from '../config';
import logger from '../utils/logger';
import { getStreamClient } from '../utils/stream';
async function getContentFeed(req, res, type, model) {
const limit = req.query.per_page || 10;
const offset = req.query.page * limit || 0;
const response = await getStreamClient()
.feed(`user_${type}`, req.params.userId)
.get({ limit, offset });
let articleIDs = response.results.map((r) => {
return r.foreign_id.split(':')[1];
});
let articles = await model.find({ _id: { $in: articleIDs } });
let articleLookup = {};
for (let a of articles) {
articleLookup[a._id] = a;
}
let sortedArticles = [];
for (let r of response.results) {
let articleID = r.foreign_id.split(':')[1];
let article = articleLookup[articleID];
if (!article) {
logger.error(
`Failed to load article ${articleID} specified in feed user_${type}:${req.params.userId}`,
);
continue;
}
sortedArticles.push(article);
}
res.json(sortedArticles);
}
exports.get = async (req, res, _) => {
const params = req.params || {};
const query = req.query || {};
if (req.User.id != params.userId) {
return res.status(404).send('Invalid user id');
}
switch (query.type) {
case 'article':
return getContentFeed(req, res, 'article', Article);
case 'episode':
return getContentFeed(req, res, 'episode', Episode);
}
res.status(400).send(
'Request must include "type" of user, timeline, article or episode',
);
};
================================================
FILE: api/src/controllers/folder.js
================================================
import mongoose from 'mongoose';
import config from '../config';
import { getStreamClient } from '../utils/stream';
import Folder from '../models/folder';
import Rss from '../models/rss';
import Podcast from '../models/podcast';
import Article from '../models/article';
import Episode from '../models/episode';
exports.list = async (req, res) => {
res.json(await Folder.find({ user: req.user.sub }));
};
exports.feed = async (req, res) => {
const folderId = req.params.folderId;
const limit = req.query.per_page || 10;
const offset = req.query.page * limit || 0;
/* Custom ranking in Stream Pro plan
// Should be defined in Stream dashboard */
const ranking = config.stream.pro
? { ranking: req.query.sort_by === 'oldest' ? 'time_oldest' : '' }
: {};
const folder = await Folder.findById(req.params.folderId);
if (!folder) return res.status(404).json({ error: 'Resource does not exist.' });
if (folder.user._id != req.user.sub) return res.sendStatus(403);
const response = await getStreamClient()
.feed('folder', folderId)
.get({ limit, offset, ...ranking });
const streamFeeds = response.results.map((r) => {
const split = r.foreign_id.split(':');
return { type: split[0], id: split[1] };
});
const articleIds = streamFeeds.filter((f) => f.type === 'articles').map((f) => f.id);
const espisodeIds = streamFeeds.filter((f) => f.type === 'episodes').map((f) => f.id);
const articles = await Article.find({ _id: { $in: articleIds } });
const episodes = await Episode.find({ _id: { $in: espisodeIds } });
const feedLookup = [...articles, ...episodes].reduce((result, feed) => {
result[feed._id] = feed;
return result;
}, {});
let feeds = [];
for (const f of streamFeeds) {
let feed = feedLookup[f.id];
if (!feed) {
logger.error(`Failed to load ${f.type}:${f.id} in folder:${folderId}`);
continue;
}
feeds.push(feed);
}
return res.json(feeds);
};
exports.get = async (req, res) => {
const folder = await Folder.findById(req.params.folderId);
if (!folder) return res.status(404).json({ error: 'Resource does not exist.' });
if (folder.user._id != req.user.sub) return res.sendStatus(403);
res.json(folder);
};
exports.post = async (req, res) => {
const data = {
user: req.user.sub,
name: req.body.name || undefined,
rss: req.body.rss || [],
podcast: req.body.podcast || [],
};
if (!data.name) return res.status(422).json({ error: 'Missing required field' });
if (!(await checkRssPodcast(data.rss, data.podcast)))
return res.status(422).json({ error: 'Some wrong feed Id provided' });
if (
(data.rss.length &&
(await Folder.find({ user: data.user, rss: { $in: data.rss } })).length) ||
(data.podcast.length &&
(await Folder.find({ user: data.user, podcast: { $in: data.podcast } }))
.length)
) {
return res.status(422).json({ error: 'Feed already has a folder' });
}
const folder = await Folder.create(data);
await streamFollowMany(folder);
res.json(await Folder.findById(folder._id));
};
exports.put = async (req, res) => {
const folderId = req.params.folderId;
const removeFeed = req.body.action === 'remove';
const name = req.body.name;
const rss = req.body.rss;
const podcast = req.body.podcast;
const user = req.user.sub;
const folder = await Folder.findById(folderId);
if (!(name || rss || podcast))
return res.status(422).json({ error: 'You have to put {name||rss||podcast}' });
if (!folder) return res.status(404).json({ error: 'Resource does not exist.' });
if (folder.user._id != user) return res.sendStatus(403);
if ((rss || podcast) && !(await checkRssPodcast([rss], [podcast])))
return res.status(404).json({ error: 'Wrong feed Id provided' });
if (removeFeed) {
await streamUnfollow(folderId, rss ? 'rss' : 'podcast', rss ? rss : podcast);
const data = rss ? { $pull: { rss } } : { $pull: { podcast } };
const removed = await Folder.findByIdAndUpdate(folderId, data, { new: true });
return res.json(removed);
}
let data = {};
if (rss) {
const prevFolder = await Folder.findOne({ user, rss });
if (prevFolder) {
await Folder.findByIdAndUpdate(prevFolder._id, { $pull: { rss } });
await streamUnfollow(prevFolder._id, 'rss', rss);
}
await streamFollow(folderId, 'rss', rss);
data = { ...data, $push: { rss } };
} else if (podcast) {
const prevFolder = await Folder.findOne({ user, podcast });
if (prevFolder) {
await Folder.findByIdAndUpdate(prevFolder._id, { $pull: { podcast } });
await streamUnfollow(prevFolder._id, 'podcast', podcast);
}
await streamFollow(folderId, 'podcast', podcast);
data = { ...data, $push: { podcast } };
}
if (name) data = { name };
const updatedFolder = await Folder.findByIdAndUpdate(folderId, data, { new: true });
res.json(updatedFolder);
};
exports.delete = async (req, res) => {
const folder = await Folder.findById(req.params.folderId);
if (!folder) return res.status(404).json({ error: 'Resource does not exist.' });
if (folder.user._id != req.user.sub) return res.sendStatus(403);
await streamUnfollowMany(folder);
await folder.remove();
res.sendStatus(204);
};
async function checkRssPodcast(rssIDs, podcastIDs) {
if (!rssIDs[0]) rssIDs = [];
if (!podcastIDs[0]) podcastIDs = [];
if (rssIDs.length) {
const rss = await Rss.find({
_id: { $in: rssIDs.map((_id) => mongoose.Types.ObjectId(_id)) },
});
if (rss.length != rssIDs.length) return false;
}
if (podcastIDs.length) {
const podcast = await Podcast.find({
_id: { $in: podcastIDs.map((_id) => mongoose.Types.ObjectId(_id)) },
});
if (podcast.length != podcastIDs.length) return false;
}
return true;
}
async function streamFollow(folderId, type, feedId) {
const feed = getStreamClient().feed('folder', folderId);
return await feed.follow(type, feedId);
}
async function streamUnfollow(folderId, type, feedId) {
const feed = getStreamClient().feed('folder', folderId);
return await feed.unfollow(type, feedId);
}
async function streamFollowMany(folder) {
const feedRels = generateRels(folder);
if (feedRels.length > 0) await getStreamClient().followMany(feedRels);
}
async function streamUnfollowMany(folder) {
const feedRels = generateRels(folder);
if (feedRels.length > 0) await getStreamClient().unfollowMany(feedRels);
}
function generateRels(folder) {
const rssRel = folder.rss.map((r) => ({
source: `folder:${folder._id}`,
target: `rss:${r._id}`,
}));
const podcastRel = folder.podcast.map((p) => ({
source: `folder:${folder._id}`,
target: `podcast:${p._id}`,
}));
return [...rssRel, ...podcastRel];
}
================================================
FILE: api/src/controllers/follow.js
================================================
import stream from 'getstream';
import async from 'async';
import mongoose from 'mongoose';
import Follow from '../models/follow';
import User from '../models/user';
import Podcast from '../models/podcast';
import RSS from '../models/rss';
import config from '../config';
import { getStreamClient } from '../utils/stream';
exports.list = async (req, res) => {
const lookup = { user: req.user.sub };
if (req.query.type === 'rss') {
lookup['rss'] = { $exists: true };
} else if (req.query.type === 'podcast') {
lookup['podcast'] = { $exists: true };
} else if (req.query.rss) {
lookup['rss'] = req.query.rss;
} else if (req.query.podcast) {
lookup['podcast'] = req.query.podcast;
} else {
throw new Error('Invalid parameter passed to follow list endpoint.');
}
const follows = await Follow.find(lookup);
return res.json(follows);
};
exports.post = async (req, res) => {
const query = req.query || {};
const user = req.user.sub;
let follow;
if (!query.type || (!query.podcast && !query.rss)) {
return res.status(422).send('Missing required type query parameter.');
}
if (query.type === 'podcast') {
if (!mongoose.Types.ObjectId.isValid(query.podcast)) {
return res.status(400).json({
error: `Parameter podcast (${query.podcast}) is an invalid ObjectId.`,
});
}
let podcast = await Podcast.findById(query.podcast);
if (!podcast) {
return res.status(404).json({ error: 'Resource not found.' });
}
follow = await Follow.getOrCreate('podcast', user, query.podcast);
} else if (query.type === 'rss') {
if (!mongoose.Types.ObjectId.isValid(query.rss)) {
return res.status(400).json({
error: `Parameter rss (${query.rss}) is an invalid ObjectId.`,
});
}
let rss = await RSS.findById(query.rss);
if (!rss) {
return res.status(404).json({ error: 'Resource not found.' });
}
follow = await Follow.getOrCreate('rss', user, query.rss);
} else {
return res.status(400).json({ error: 'Missing required parameter.' });
}
return res.json(follow);
};
exports.delete = async (req, res) => {
const query = req.query || {};
const lookup = { user: req.user.sub };
if (query.rss) {
lookup['rss'] = query.rss;
} else if (query.podcast) {
lookup['podcast'] = query.podcast;
} else {
return res
.send(400)
.json({ error: 'Invalid parameter passed to delete method.' });
}
const follow = await Follow.findOne(lookup);
if (follow) {
await follow.removeFromStream();
await follow.remove();
}
return res.status(204).send();
};
================================================
FILE: api/src/controllers/health.js
================================================
import Article from '../models/article';
import Episode from '../models/episode';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import moment from 'moment';
import config from '../config';
import { Throw } from '../utils/errors';
import Queue from 'bull';
import Arena from 'bull-arena';
import logger from '../utils/logger';
let version;
if (process.env.DOCKER) {
version = 'DOCKER';
} else {
version = require('../../../app/package.json').version;
}
const rssQueue = new Queue('rss', config.cache.uri);
const ogQueue = new Queue('og', config.cache.uri);
const podcastQueue = new Queue('podcast', config.cache.uri);
const socialQueue = new Queue('socail', config.cache.uri);
const streamQueue = new Queue('stream', config.cache.uri);
const tooOld = 3 * 60 * 60 * 1000;
const queues = {
'RSS Queue': rssQueue,
'OG Queue': ogQueue,
'Podcast Queue': podcastQueue,
'Social Score Queue': socialQueue,
'Personalisation-sync Queue': streamQueue,
};
const queueTTL = 24 * 60 * 60 * 1000; // 1 day
const queueCompletedCleanup = async (queue) => {
await queue.clean(queueTTL, 'completed'); // cleans all jobs that completed over 1 day ago.
};
const queueFailedCleanup = async (queue) => {
await queue.clean(queueTTL, 'failed'); // clean all jobs that failed over 1 day ago
};
queueCompletedCleanup(rssQueue);
queueCompletedCleanup(ogQueue);
queueCompletedCleanup(podcastQueue);
queueCompletedCleanup(socialQueue);
queueCompletedCleanup(streamQueue);
queueFailedCleanup(rssQueue);
queueFailedCleanup(ogQueue);
queueFailedCleanup(podcastQueue);
queueFailedCleanup(socialQueue);
queueFailedCleanup(streamQueue);
exports.health = async (req, res) => {
res.status(200).send({ version, healthy: '100%' });
};
exports.status = async (req, res) => {
const output = { version, code: 200, rss: {}, podcast: {} };
const latestArticle = await Article.findOne().sort({ _id: -1 });
const latestEpisode = await Episode.findOne().sort({ _id: -1 });
const now = new Date();
output.now = now;
if (latestArticle) {
output.mostRecentArticle = moment(latestArticle.createdAt).fromNow();
if (now - latestArticle.createdAt > tooOld) {
output.code = 500;
output.error = 'The most recent article is too old.';
}
} else {
output.mostRecentArticle = -1;
}
if (latestEpisode) {
output.mostRecentEpisode = moment(latestEpisode.createdAt).fromNow();
if (now - latestEpisode.createdAt > tooOld) {
output.code = 500;
output.error = 'The most recent episode is too old.';
}
} else {
output.mostRecentEpisode = -1;
}
output.rss.parsing = await RSS.count({ 'queueState.isParsing': true });
output.rss.og = await RSS.count({ 'queueState.isUpdatingOG': true });
output.rss.stream = await RSS.count({ 'queueState.isSynchronizingWithStream': true });
output.rss.social = await RSS.count({ 'queueState.isFetchingSocialScore': true });
output.podcast.parsing = await Podcast.count({ 'queueState.isParsing': true });
output.podcast.og = await Podcast.count({ 'queueState.isUpdatingOG': true });
output.podcast.stream = await Podcast.count({
'queueState.isSynchronizingWithStream': true,
});
if (output.rss.parsing > 2000) {
output.code = 500;
output.error = `There are too many RSS feeds currently parsing ${output.rssCurrentlyParsing}`;
}
if (output.podcast.parsing > 500) {
output.code = 500;
output.error = `There are too many Podcast feeds currently parsing ${output.podcastCurrentlyParsing}`;
}
res.status(output.code).json(output);
};
exports.queue = async (req, res) => {
let output = { version, code: 200 };
for (const [key, queue] of Object.entries(queues)) {
let queueStatus = await queue.getJobCounts();
output[key] = queueStatus;
if (queueStatus.waiting > 2500) {
output.code = 500;
output.error = `Queue ${key} has more than 2500 items waiting to be processed: ${queueStatus.waiting} are waiting`;
}
}
res.status(output.code).send(output);
};
exports.sentryThrow = async (req, res) => {
Throw();
};
exports.sentryLog = async (req, res) => {
try {
Throw();
} catch (err) {
logger.error('this is a test error', {
err,
tags: { env: 'testing' },
extra: { additional: 'data', is: 'awesome' },
});
}
try {
Throw();
} catch (err) {
logger.error({ err });
}
logger.error('0');
logger.error('1');
res.status(200).send('{}');
};
exports.bullArena = Arena(
{
Bull: Queue,
queues: [
{
type: 'bull',
hostId: 'local',
name: 'rss',
url: config.cache.uri,
},
{
type: 'bull',
hostId: 'local',
name: 'podcast',
url: config.cache.uri,
},
{
type: 'bull',
hostId: 'local',
name: 'og',
url: config.cache.uri,
},
{
type: 'bull',
hostId: 'local',
name: 'social',
url: config.cache.uri,
},
{
type: 'bull',
hostId: 'local',
name: 'stream',
url: config.cache.uri,
},
],
},
{ disableListen: true },
);
================================================
FILE: api/src/controllers/listen.js
================================================
import Listen from '../models/listen';
import User from '../models/user';
import { trackEngagement } from '../utils/analytics';
import config from '../config';
exports.list = async (req, res) => {
if (req.query.user && req.query.user != req.User.id) {
return res.sendStatus(403);
}
const listens = await Listen.find({
user: req.query.user,
episode: req.query.episode,
});
res.json(listens);
};
exports.post = async (req, res) => {
const data = Object.assign({}, req.body, { user: req.user.sub });
const { _id, id, ...cleanedData } = data;
const listen = await Listen.findOneAndUpdate(
{ user: cleanedData.user, episode: cleanedData.episode },
{ $set: cleanedData },
{ new: true, upsert: true },
);
const duration = data.duration;
if (Math.floor(duration / 15) % 4 == 0) {
const foreign_id = `episode:${data.episode}`;
await trackEngagement(req.User, {
label: 'listen_progress',
content: { foreign_id: foreign_id },
});
}
res.json(listen);
};
================================================
FILE: api/src/controllers/note.js
================================================
import Note from '../models/note';
exports.list = async (req, res) => {
res.json(await Note.find({ user: req.user.sub }).sort({ updatedAt: -1 }));
};
exports.get = async (req, res) => {
const note = await Note.findById(req.params.noteId);
if (!note) return res.status(404).json({ error: 'Resource does not exist.' });
if (note.user._id != req.user.sub) return res.sendStatus(403);
res.json(note);
};
exports.post = async (req, res) => {
const data = {
user: req.user.sub,
article: req.body.article,
episode: req.body.episode,
start: req.body.start,
end: req.body.end,
text: req.body.text || '',
};
if (data.start == undefined || data.end == undefined)
return res.status(422).json({ error: 'missing start|end offset' });
if (!data.article && !data.episode)
return res.status(422).json({ error: 'missing article||episode id' });
if (data.article && data.episode)
return res.status(422).json({ error: 'both article||episode id' });
const overlaps = await Note.find({
user: data.user,
article: data.article,
episode: data.episode,
$nor: [{ end: { $lte: data.start } }, { start: { $gte: data.end } }],
})
.sort({ start: 1 })
.lean();
const mergedNotes = overlaps.map((n) => n._id);
if (overlaps.length) {
for (const note of overlaps) {
if (note.start < data.start) data.start = note.start;
if (note.end > data.end) data.end = note.end;
if (note.text) data.text = data.text + '\n' + note.text;
}
await Note.deleteMany({ _id: { $in: mergedNotes } });
}
const note = await Note.create(data);
const noteJson = (await Note.findById(note._id)).toJSON();
res.json({ ...noteJson, mergedNotes });
};
exports.put = async (req, res) => {
const noteId = req.params.noteId;
const note = await Note.findById(noteId).lean();
if (!note) return res.status(404).json({ error: 'Resource does not exist.' });
if (note.user._id != req.user.sub) return res.sendStatus(403);
const start = req.body.start || note.start;
const end = req.body.end || note.end;
const text = req.body.text || note.text || '';
res.json(await Note.findByIdAndUpdate(noteId, { start, end, text }, { new: true }));
};
exports.delete = async (req, res) => {
const note = await Note.findById(req.params.noteId);
if (!note) return res.status(404).json({ error: 'Resource does not exist.' });
if (note.user._id != req.user.sub) return res.sendStatus(403);
await note.remove();
res.sendStatus(204);
};
================================================
FILE: api/src/controllers/opml.js
================================================
import opmlParser from 'node-opml-parser';
import opmlGenerator from 'opml-generator';
import moment from 'moment';
import entities from 'entities';
import normalizeUrl from 'normalize-url';
import stream from 'getstream';
import util from 'util';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import Follow from '../models/follow';
import User from '../models/user';
import config from '../config';
import { isBlockedURLs } from '../utils/blockedURLs';
import * as rateLimit from '../utils/rate-limiter';
import { RssQueueAdd, PodcastQueueAdd } from '../asyncTasks';
import { IsPodcastURL } from '../parsers/detect-type';
import search from '../utils/search';
import { isURL } from '../utils/validation';
exports.get = async (req, res) => {
let follows = await Follow.find({ user: req.user.sub });
let user = await User.find({ _id: req.user.sub });
if (!user) {
return res.status(404).json({ error: 'User does not exist.' });
}
let header = {
dateCreated: moment().toISOString(),
ownerName: user.name,
title: `Subscriptions in Winds - Powered by ${config.product.author}`,
};
let outlines = follows.map((follow) => {
let feed = follow.rss ? follow.rss : follow.podcast;
let feedType = follow.rss ? 'rss' : 'podcast';
let obj = {
htmlUrl: feed.url,
title: feed.title,
type: feedType,
xmlUrl: feed.feedUrl,
};
return obj;
});
let opml = opmlGenerator(header, outlines);
res.set({
'Content-Disposition': 'attachment; filename=export.opml;',
'Content-Type': 'application/xml',
});
res.end(opml);
};
function partitionBy(collection, selector) {
if (!collection.length) {
return [];
}
const partitions = [[collection[0]]];
let currentPartition = 0;
let lastElement = selector(collection[0]);
for (let i = 1; i < collection.length; ++i) {
const element = selector(collection[i]);
if (element !== lastElement) {
partitions.push([]);
++currentPartition;
}
partitions[currentPartition].push(collection[i]);
lastElement = element;
}
return partitions;
}
async function identifyFeedType(feed) {
let schema;
let publicationType;
let isPodcast;
if (!feed.valid) {
throw new Error(`Invalid feedUrl ${feed.feedUrl}`);
}
try {
isPodcast = await IsPodcastURL(feed.feedUrl);
} catch (_) {
throw new Error(`Error opening ${feed.feedUrl}`);
}
if (isPodcast) {
schema = Podcast;
publicationType = 'podcast';
} else {
schema = RSS;
publicationType = 'rss';
}
const feedUrl = normalizeUrl(feed.feedUrl);
if (!isURL(feedUrl)) {
throw new Error(`Invalid URL for OPML import ${feedUrl}`);
}
return { feed, schema, publicationType, url: feedUrl };
}
async function getOrCreateManyPublications(feeds) {
if (!feeds.length) {
return [];
}
const feedUrls = feeds.map((p) => p.url);
const instances = await feeds[0].schema.find({ feedUrl: { $in: feedUrls } });
const existingFeedUrls = new Set(instances.map((i) => i.feedUrl));
const newPublications = feeds.filter((p) => !existingFeedUrls.has(p.url));
if (!newPublications.length) {
return instances;
}
const newInstanceData = newPublications.map((p) => {
const title = entities.decodeHTML(p.feed.title) || '';
return {
categories: p.publicationType,
description: title.substring(0, 240),
favicon: p.feed.favicon,
feedUrl: p.url,
lastScraped: moment().subtract(12, 'hours'),
public: true,
publicationDate: moment().toISOString(),
title,
url: p.feed.url,
};
});
const newInstances = await feeds[0].schema.insertMany(newInstanceData);
const queue =
feeds[0].publicationType.toLowerCase() == 'rss' ? RssQueueAdd : PodcastQueueAdd;
const queueData = newInstances.map((i) => ({
[i.categories]: i._id,
url: i.feedUrl,
}));
const enqueues = queueData.map((d) =>
queue(d, { priority: 1, removeOnComplete: true, removeOnFail: true }),
);
const indexing = newInstances.map((i) => search(i.searchDocument()));
await Promise.all(enqueues.concat(indexing));
return instances.concat(newInstances);
}
exports.post = async (req, res) => {
const upload = Buffer.from(req.file.buffer).toString('utf8');
if (!upload) {
return res.status(422).json({ error: 'Invalid OPML upload.' });
}
let feeds;
try {
feeds = await util.promisify(opmlParser)(upload);
} catch (e) {
return res.status(422).json({ error: 'Invalid OPML upload.' });
}
for (const feed of feeds) {
feed.valid = true;
if (isURL(feed.feedUrl)) {
feed.feedUrl = normalizeUrl(feed.feedUrl).trim();
} else {
feed.valid = false;
}
if (isURL(feed.url)) {
feed.url = normalizeUrl(feed.url);
}
feed.favicon = '';
if (feeds.site && feeds.site.favicon) {
feed.favicon = feeds.site.favicon;
}
}
const feedIdentities = await Promise.all(
feeds.map(async (f) => {
try {
if (isBlockedURLs(feeds.feedUrl)) {
return { feedUrl: f.feedUrl, error: "this feed can't be added" };
}
return { result: await identifyFeedType(f) };
} catch (err) {
return { feedUrl: f.feedUrl, error: err.message };
}
}),
);
const failedFeeds = feedIdentities.filter((f) => !!f.error);
const feedSchemas = feedIdentities.filter((f) => !f.error).map((f) => f.result);
feedSchemas.sort((lhs, rhs) =>
lhs.publicationType.localeCompare(rhs.publicationType),
);
//XXX: process podcasts first, then rss to allow bulk operations
const partitions = partitionBy(feedSchemas, (p) => p.schema);
let publications = [];
const chunkSize = 1000;
for (const feeds of partitions) {
for (let offset = 0; offset < feeds.length; offset += chunkSize) {
const limit = offset + chunkSize;
const chunk = feeds.slice(offset, limit);
publications = publications.concat(await getOrCreateManyPublications(chunk));
}
}
let follows = [];
for (let offset = 0; offset < publications.length; offset += chunkSize) {
const limit = offset + chunkSize;
const chunk = publications.slice(offset, limit);
await rateLimit.tick(req.user.sub);
const followInstructions = chunk.map((p) => ({
type: p.categories.toLowerCase(),
userID: req.user.sub,
publicationID: p._id,
}));
const newFollows = await Follow.getOrCreateMany(followInstructions);
follows = follows.concat(
newFollows.map((f, i) => ({ feedUrl: chunk[i].url, follow: f })),
);
}
const errors = failedFeeds.map((f) => ({ ...f, follow: {} }));
return res.json(follows.concat(errors));
};
================================================
FILE: api/src/controllers/pin.js
================================================
import mongoose from 'mongoose';
import Pin from '../models/pin';
import config from '../config';
import { trackEngagement } from '../utils/analytics';
import { getStreamClient } from '../utils/stream';
exports.list = async (req, res) => {
const query = req.query || {};
if (query.type === 'episode' || query.type === 'article') {
let obj = {};
obj[query.type] = { $exists: true };
obj['user'] = req.user.sub; // can only list pins for the
return res.json(await Pin.find(obj).sort({ _id: -1 }));
}
res.json(await Pin.apiQuery(req.query));
};
exports.get = async (req, res) => {
if (!mongoose.Types.ObjectId.isValid(req.params.pinId)) {
return res.status(400).json({
error: `Resource pinId (${req.params.pinId}) is an invalid ObjectId.`,
});
}
let pin = await Pin.findById(req.params.pinId);
if (!pin) {
return res.status(404).json({ error: 'Resource does not exist.' });
}
res.json(pin);
};
exports.post = async (req, res) => {
const data = Object.assign({}, req.body, { user: req.user.sub });
let type;
if (data.hasOwnProperty('article')) {
type = 'article';
} else if (data.hasOwnProperty('episode')) {
type = 'episode';
} else {
return res.status(422).json({
error: 'Missing required fields.',
});
}
const obj = { user: data.user, [type]: data[type] };
if (!!(await Pin.findOne(obj))) {
return res.status(409).json({ error: 'Resource already exists.' });
}
const pin = await Pin.create(data);
await getStreamClient()
.feed('user', pin.user)
.addActivity({
actor: pin.user,
verb: 'pin',
object: pin._id,
foreign_id: `pins:${pin._id}`,
time: pin.createdAt,
});
const label = pin.article ? 'pin_article' : 'pin_episode';
const foreignId = pin.article ? `article:${pin.article}` : `episode:${pin.episode}`;
await trackEngagement(req.User, {
label: label,
content: { foreign_id: foreignId },
});
res.json(await Pin.findOne({ _id: pin._id }));
};
exports.delete = async (req, res) => {
const exists = await Pin.findOne({ _id: req.params.pinId, user: req.user.sub });
if (!exists) {
return res.status(404).json({ error: 'Resource does not exist.' });
}
await Pin.remove({ _id: req.params.pinId });
res.sendStatus(204);
};
================================================
FILE: api/src/controllers/playlist.js
================================================
import Playlist from '../models/playlist';
import config from '../config';
import search from '../utils/search';
async function listFilter(req, res) {
const playlists = await Playlist.apiQuery(req.query);
res.json(playlists);
}
exports.list = async (req, res, _) => {
const query = req.query || {};
if (query.user && query.user != req.User.id) {
return res.sendStatus(403);
}
await listFilter(req, res);
};
exports.get = async (req, res, _) => {
if (req.params.playlistId == 'undefined') {
return res.sendStatus(404);
}
const playlist = await Playlist.findById(req.params.playlistId);
if (!playlist) {
return res.sendStatus(404);
}
if (playlist.user._id != req.User.id) {
return res.sendStatus(403);
}
res.json(playlist);
};
exports.post = async (req, res, _) => {
const data = Object.assign({}, req.body, { user: req.user.sub });
let playlist;
playlist = await Playlist.create(data);
playlist = await Playlist.findById(playlist._id);
await search({
_id: playlist._id,
episodes: playlist.episodes,
name: playlist.name,
type: 'playlist',
user: playlist.user,
});
res.json(playlist);
};
exports.put = async (req, res, _) => {
const playlist = await Playlist.findById(req.params.playlistId);
if (!playlist) {
return res.sendStatus(404);
}
if (playlist.user._id != req.User.id) {
return res.sendStatus(403);
}
await Playlist.update({ _id: req.params.playlistId }, req.body, { new: true });
res.json(await Playlist.findOne({ _id: req.params.playlistId }));
};
exports.delete = async (req, res, _) => {
const playlist = await Playlist.findById(req.params.playlistId);
if (!playlist) {
return res.sendStatus(404);
}
if (playlist.user._id != req.User.id) {
return res.sendStatus(403);
}
await Playlist.remove({ _id: req.params.playlistId });
res.sendStatus(204);
};
================================================
FILE: api/src/controllers/podcast.js
================================================
import mongoose from 'mongoose';
import normalizeUrl from 'normalize-url';
import Podcast from '../models/podcast';
import { isURL } from '../utils/validation';
import { isBlockedURLs } from '../utils/blockedURLs';
import { discoverRSS } from '../parsers/discovery';
import { getPodcastRecommendations } from '../utils/personalization';
import { ParsePodcast } from '../parsers/feed';
import strip from 'strip';
import search from '../utils/search';
import { PodcastQueueAdd, OgQueueAdd } from '../asyncTasks';
exports.list = async (req, res) => {
let query = req.query || {};
let podcasts;
if (query.type === 'recommended') {
podcasts = await getPodcastRecommendations(req.User._id.toString(), 7);
} else {
podcasts = await Podcast.apiQuery(req.query);
}
res.json(podcasts);
};
exports.get = async (req, res) => {
const podcastId = req.params.podcastId;
if (!mongoose.Types.ObjectId.isValid(podcastId)) {
return res
.status(422)
.json({ error: `Podcast ID ${podcastId} is an invalid ObjectId.` });
}
let podcast = await Podcast.findById(podcastId).exec();
if (!podcast) {
return res.status(404).json({ error: `Resource not found.` });
}
res.json(podcast.serialize());
};
exports.post = async (req, res) => {
const data = Object.assign(req.body, { user: req.user.sub }) || {};
// todo refactor this check for validating partial urls like google.com
let url;
try {
url = normalizeUrl(data.feedUrl);
} catch (e) {
return res.status(400).json({ error: 'Please provide a valid podcast URL.' });
}
if (!data.feedUrl || !isURL(url)) {
return res.status(400).json({ error: 'Please provide a valid podcast URL.' });
}
if (isBlockedURLs(data.feedUrl)) {
return res.status(400).json({ error: 'This podcast can not be added.' });
}
let foundPodcasts = await discoverRSS(normalizeUrl(data.feedUrl));
if (!foundPodcasts.feedUrls.length) {
return res.status(404).json({ error: `Can't find any podcasts.` });
}
let insertedPodcasts = [];
let podcasts = [];
for (let feed of foundPodcasts.feedUrls.slice(0, 10)) {
let podcastContent = await ParsePodcast(feed.url, 1);
let title, url, images, description;
if (podcastContent) {
title = strip(podcastContent.title) || strip(feed.title);
url = podcastContent.link || foundPodcasts.site.url;
images = {
favicon: foundPodcasts.site.favicon,
og: podcastContent.image,
};
description = podcastContent.description;
} else {
title = strip(feed.title);
url = foundPodcasts.site.url;
images = { favicon: foundPodcasts.site.favicon };
description = '';
}
let feedUrl = normalizeUrl(feed.url);
if (!isURL(feedUrl)) {
continue;
}
let podcast;
podcast = await Podcast.findOne({ feedUrl: feedUrl });
if (!podcast || (podcast && !podcast.featured)) {
let response = await Podcast.findOneAndUpdate(
{ feedUrl: feedUrl },
{
categories: 'podcast',
description: (description || '').substring(0, 240),
feedUrl: feedUrl,
images: images,
lastScraped: new Date(0),
title: title,
url: normalizeUrl(url),
valid: true,
},
{
new: true,
rawResult: true,
upsert: true,
},
);
podcast = response.value;
if (response.lastErrorObject.upserted) {
insertedPodcasts.push(podcast);
}
}
podcasts.push(podcast);
}
let promises = [];
insertedPodcasts.map((p) => {
let scrapingPromise = PodcastQueueAdd(
{
podcast: p._id,
url: p.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
promises.push(scrapingPromise);
if (!p.images.og && p.link) {
promises.push(
OgQueueAdd(
{
url: p.link,
podcast: p._id,
type: 'podcast',
},
{
removeOnComplete: true,
removeOnFail: true,
},
),
);
}
promises.push(search(p.searchDocument()));
});
await Promise.all(promises);
res.status(200).json(
podcasts.map((p) => {
return p.serialize();
}),
);
};
exports.put = async (req, res) => {
const podcastId = req.params.podcastId;
if (!mongoose.Types.ObjectId.isValid(podcastId)) {
return res
.status(422)
.json({ error: `Podcast ID ${podcastId} is an invalid ObjectId.` });
}
if (!req.User.admin) {
return res.status(403).send();
}
if (!podcastId) {
return res.status(401).json({ error: 'Missing required Podcast ID.' });
}
let podcast = await Podcast.findByIdAndUpdate({ _id: podcastId }, req.body, {
new: true,
});
if (!podcast) {
return res.status(404).json({ error: 'Podcast could not be found.' });
}
res.json(podcast);
};
================================================
FILE: api/src/controllers/rss.js
================================================
import mongoose from 'mongoose';
import moment from 'moment';
import normalizeUrl from 'normalize-url';
import entities from 'entities';
import RSS from '../models/rss';
import { discoverRSS } from '../parsers/discovery';
import search from '../utils/search';
import { isBlockedURLs } from '../utils/blockedURLs';
import { isURL } from '../utils/validation';
import { RssQueueAdd, OgQueueAdd } from '../asyncTasks';
import { getRSSRecommendations } from '../utils/personalization';
exports.list = async (req, res) => {
const query = req.query || {};
let rss;
if (query.type === 'recommended') {
rss = await getRSSRecommendations(req.User._id.toString(), 7);
} else {
rss = await RSS.apiQuery(req.query);
}
res.json(rss);
};
exports.get = async (req, res) => {
const rssId = req.params.rssId;
if (!mongoose.Types.ObjectId.isValid(rssId)) {
return res.status(422).json({ error: `RSS ID ${rssId} is invalid.` });
}
let rss = await RSS.findById(rssId).exec();
if (!rss) {
return res.sendStatus(404);
}
res.json(rss.serialize());
};
exports.post = async (req, res) => {
const data = req.body || {};
let normalizedUrl;
// TODO: refactor this url check in utitlies
try {
normalizedUrl = normalizeUrl(data.feedUrl);
} catch (e) {
return res.status(400).json({ error: 'Please provide a valid RSS URL.' });
}
if (!data.feedUrl || !isURL(normalizedUrl)) {
return res.status(400).json({ error: 'Please provide a valid RSS URL.' });
}
if (isBlockedURLs(data.feedUrl)) {
return res.status(400).json({ error: 'This feed can not be added.' });
}
let foundRSS = await discoverRSS(normalizeUrl(data.feedUrl));
if (!foundRSS.feedUrls.length) {
return res
.status(404)
.json({ error: "We couldn't find any feeds for that RSS feed URL :(" });
}
let insertedFeeds = [];
let feeds = [];
for (let feed of foundRSS.feedUrls.slice(0, 10)) {
let feedTitle = feed.title;
if (!feedTitle) {
continue;
}
if (feedTitle.toLowerCase() === 'rss') {
feedTitle = foundRSS.site.title;
}
let feedUrl = normalizeUrl(feed.url);
if (!isURL(feedUrl)) {
continue;
}
let rss = await RSS.findOne({ feedUrl: feedUrl });
const limit = moment().subtract(30, 'seconds');
// don't update featured RSS feeds since that ends up removing images etc
if (!rss || (!rss.featured && limit.isAfter(rss.lastScraped))) {
let response = await RSS.findOneAndUpdate(
{ feedUrl: feedUrl },
{
categories: 'RSS',
description: (entities.decodeHTML(feed.title) || '').substring(
0,
240,
),
feedUrl: feedUrl,
images: {
favicon: foundRSS.site.favicon,
},
lastScraped: moment().format(),
title: entities.decodeHTML(feedTitle),
url: foundRSS.site.url,
valid: true,
},
{
new: true,
rawResult: true,
upsert: true,
},
);
rss = response.value;
if (response.lastErrorObject.upserted) {
insertedFeeds.push(rss);
}
}
feeds.push(rss);
}
let promises = [];
insertedFeeds.map((f) => {
promises.push(search(f.searchDocument()));
let rssScrapingPromise = RssQueueAdd(
{
rss: f._id,
url: f.feedUrl,
},
{
priority: 1,
removeOnComplete: true,
removeOnFail: true,
},
);
promises.push(rssScrapingPromise);
if (!f.images.og && f.url) {
let ogPromise = OgQueueAdd(
{
url: f.url,
rss: f._id,
type: 'rss',
},
{
removeOnComplete: true,
removeOnFail: true,
},
);
promises.push(ogPromise);
}
});
await Promise.all(promises);
res.status(201);
res.json(
feeds.map((f) => {
return f.serialize();
}),
);
};
exports.put = async (req, res) => {
if (!req.User.admin) {
return res
.status(403)
.json({ error: 'You must be an admin to perform this action.' });
}
if (!req.params.rssId) {
return res
.status(401)
.json({ error: 'You must provide a valid RSS ID to perform this action' });
}
let rss = await RSS.findByIdAndUpdate(
{
_id: req.params.rssId,
},
req.body,
{ new: true },
);
if (!rss) {
return res
.status(404)
.json({ error: `Can't find RSS feed with id ${req.params.rssId}` });
}
res.json(rss);
};
================================================
FILE: api/src/controllers/tag.js
================================================
import mongoose from 'mongoose';
import Tag from '../models/tag';
exports.list = async (req, res) => {
res.json(await Tag.find({ user: req.user.sub }));
};
exports.get = async (req, res) => {
const tag = await Tag.findById(req.params.tagId);
if (!tag) return res.status(404).json({ error: 'Resource does not exist.' });
if (tag.user._id != req.user.sub) return res.sendStatus(403);
res.json(tag);
};
exports.post = async (req, res) => {
const data = {
user: req.user.sub,
name: req.body.name,
article: req.body.article || [],
episode: req.body.episode || [],
};
if (!data.name) return res.status(422).json({ error: 'Missing required field' });
const tag = await Tag.create(data);
res.json(await Tag.findById(tag._id));
};
exports.put = async (req, res) => {
const tagId = req.params.tagId;
const name = req.body.name;
const article = req.body.article;
const episode = req.body.episode;
const user = req.user.sub;
const remove = req.body.action === 'remove';
if (!(name || article || episode))
return res.status(422).json({ error: 'You have to put data' });
if (article && episode) return res.status(422).json({ error: 'On Feed at a time!' });
const tag = await Tag.findById(tagId).lean();
if (!tag) return res.status(404).json({ error: 'Resource does not exist.' });
if (tag.user._id != user) return res.sendStatus(403);
let data = {};
if (remove) data = article ? { $pull: { article } } : { $pull: { episode } };
else data = article ? { $addToSet: { article } } : { $addToSet: { episode } };
if (name) data = { ...data, name };
await Tag.findByIdAndUpdate(tagId, data);
res.json(await Tag.findById(tagId));
};
exports.delete = async (req, res) => {
const tag = await Tag.findById(req.params.tagId);
if (!tag) return res.status(404).json({ error: 'Resource does not exist.' });
if (tag.user._id != req.user.sub) return res.sendStatus(403);
await tag.remove();
res.sendStatus(204);
};
================================================
FILE: api/src/controllers/user.js
================================================
import validator from 'validator';
import User from '../models/user';
import RSS from '../models/rss';
import Podcast from '../models/podcast';
import personalization from '../utils/personalization';
exports.list = async (req, res) => {
let users = [];
if (req.query.type === 'recommended') {
let recommendedUserIds = await personalization({
endpoint: '/winds_user_recommendations',
userId: req.user.sub,
});
users = await User.find({ _id: { $in: recommendedUserIds } });
} else {
users = await User.apiQuery(req.query).select(
'name email username bio url twitter background admin',
);
}
res.json(users);
};
exports.delete = async (req, res) => {
if (req.params.userId !== req.user.sub) {
return res.sendStatus(403);
}
await req.User.remove();
res.sendStatus(204);
};
exports.get = async (req, res) => {
if (req.params.user == 'undefined') {
return res.sendStatus(404);
}
let user = await User.findById(req.params.userId);
if (!user) {
return res.status(404).send('User not found');
}
user.password = undefined;
user.recoveryCode = undefined;
let serialized = user;
if (user._id.toString() === req.user.sub) {
serialized = user.serializeAuthenticatedUser();
}
res.json(serialized);
};
exports.put = async (req, res) => {
if (req.params.userId !== req.user.sub) {
return res.status(403).json({ error: 'Access denied.' });
}
const data = req.body || {};
if (data.email && !validator.isEmail(data.email)) {
return res.status(400).json({ error: 'Invalid or malformed email address.' });
}
const regex = /^[\w-]+$/;
if (data.username && !regex.test(data.username)) {
return res.status(400).json({
error: 'Usernames must be alphanumeric but can only contain _, . or -.',
});
}
let user = await User.findById(req.params.userId);
if (!user) {
return res.sendStatus(404);
}
if (data.username) {
let userByUsername = await User.findOne({ username: data.username });
if (userByUsername && userByUsername.id != user.id) {
return res
.status(409)
.json({ error: 'A resource with this username already exists' });
}
}
if (data.email) {
let userByEmail = await User.findOne({ email: data.email });
if (userByEmail && userByEmail.email != user.email) {
return res.status(409).send('User with this email already exists');
}
}
const whitelist = Object.assign(
{},
...[
'name',
'email',
'username',
'password',
'interests',
'bio',
'url',
'twitter',
'background',
'preferences',
'recoveryCode',
'active',
].map((key) => ({
[key]: data[key],
})),
);
user = await User.findByIdAndUpdate({ _id: req.params.userId }, data, {
new: true,
});
user.password = undefined;
user.recoveryCode = undefined;
res.status(201).json(user);
};
================================================
FILE: api/src/fixtures/featured.json
================================================
{
"podcasts": [
{
"category": "UI/UX",
"name": "Design Details",
"feedUrl": "https://rss.simplecast.com/podcasts/1034/rss",
"site": "https://spec.fm/podcasts/design-details"
},
{
"category": "UI/UX",
"name": "Layout.FM",
"feedUrl": "http://layout.fm/rss",
"site": "http://layout.fm/"
},
{
"category": "UI/UX",
"name": "Responsive Web Design",
"feedUrl": "https://responsivewebdesign.com/podcast/feed.xml",
"site": "https://responsivewebdesign.com/podcast/"
},
{
"category": "UI/UX",
"name": "Creative Coding Podcast",
"feedUrl": "http://creativecodingpodcast.com/feed/",
"site": "http://creativecodingpodcast.com/"
},
{
"category": "UI/UX",
"name": "UX Pod",
"feedUrl": "http://uxpod.libsyn.com/rss",
"site": "http://uxpod.com/"
},
{
"category": "UI/UX",
"name": "Build and Launch",
"feedUrl": "https://rss.simplecast.com/podcasts/323/rss",
"site": "https://buildandlaunch.net/"
},
{
"category": "UI/UX",
"name": "Nice to meet you",
"feedUrl":
"http://feeds.soundcloud.com/users/soundcloud:users:5984399/sounds.rss",
"site": "http://www.vanschneider.com/show"
},
{
"category": "UI/UX",
"name": "Design Review",
"feedUrl": "http://www.designreviewpodcast.com/design-review.rss",
"site": "http://www.designreviewpodcast.com/"
},
{
"category": "UI/UX",
"name": "Overtime",
"feedUrl": "https://rss.simplecast.com/podcasts/1515/rss",
"site": "https://dribbble.com/overtime"
},
{
"category": "UI/UX",
"name": "DesignBetter",
"feedUrl": "http://designbetter.libsyn.com/rss",
"site": "https://www.designbetter.co/podcast"
},
{
"category": "UI/UX",
"name": "High Resolution",
"feedUrl": "https://rss.simplecast.com/podcasts/2652/rss",
"site": "https://www.highresolution.design/"
},
{
"category": "UI/UX",
"name": "UI Breakfast",
"feedUrl": "https://rss.simplecast.com/podcasts/1441/rss",
"site": "https://uibreakfast.com/category/podcast/"
},
{
"category": "Startups & VC",
"name": "a16z",
"feedUrl": "https://a16z.com/feed/",
"site": "https://a16z.com/"
},
{
"category": "Startups & VC",
"name": "To The Top",
"feedUrl": "http://nathanlatkathetop.libsyn.com/rss",
"site": "http://nathanlatkathetop.libsyn.com"
},
{
"category": "Startups & VC",
"name": "Rocketship.fm",
"feedUrl":
"https://www.omnycontent.com/d/playlist/20f38a10-1afe-4760-bb32-a7dd01317ee4/cfecce2e-033b-4f28-a323-a7de003f116b/b76fb826-9e99-4328-b3fa-a7de003f1179/podcast.rss",
"site": "http://rocketship.fm/"
},
{
"category": "Startups & VC",
"name": "Twenty Minute VC",
"feedUrl": "http://thetwentyminutevc.libsyn.com/rss",
"site": "http://www.thetwentyminutevc.com/"
},
{
"category": "Startups & VC",
"name": "This Week in Startups",
"feedUrl":
"http://feeds.soundcloud.com/users/soundcloud:users:6888303/sounds.rss",
"site": "https://thisweekinstartups.com/"
},
{
"category": "Startups & VC",
"name": "Acquired.fm",
"feedUrl": "http://www.acquired.fm/episodes?format=rss",
"site": "http://www.acquired.fm/"
},
{
"category": "Startups & VC",
"name": "The Impact Podcast",
"feedUrl": "http://feeds.feedburner.com/soundcloud/ZGMH",
"site": "https://georgianpartners.com/category/podcast/"
},
{
"category": "Startups & VC",
"name": "Startup Podcast",
"feedUrl": "http://feeds.hearstartup.com/hearstartup",
"site": "https://www.gimletmedia.com/startup/"
},
{
"category": "Startups & VC",
"name": "EntreLeadership",
"feedUrl": "http://entreleadershippodcast.ramsey.libsynpro.com/rss",
"site": "https://www.entreleadership.com/blog/podcast"
},
{
"category": "Startups & VC",
"name": "Founders Talk",
"feedUrl": "https://changelog.com/founderstalk/feed",
"site": "https://changelog.com/founderstalk"
},
{
"category": "Startups & VC",
"name": "Startups for the Rest of Us",
"feedUrl": "http://www.startupsfortherestofus.com/feed",
"site": "http://www.startupsfortherestofus.com/"
},
{
"category": "Programming",
"name": "Merge Conflict",
"feedUrl": "http://www.mergeconflict.fm/rss",
"site": "http://www.mergeconflict.fm/"
},
{
"category": "Programming",
"name": "Software Engineering Daily",
"feedUrl": "http://softwareengineeringdaily.com/feed/podcast/",
"site": "https://softwareengineeringdaily.com/"
},
{
"category": "Programming",
"name": "Cynical Developer",
"feedUrl": "https://cynicaldeveloper.com/feed/podcast/",
"site": "https://cynicaldeveloper.com/Podcast/"
},
{
"category": "Programming",
"name": "Changelog",
"feedUrl": "https://changelog.com/podcast/feed",
"site": "https://changelog.com"
},
{
"category": "Programming",
"name": "Go Time",
"feedUrl": "https://changelog.com/gotime/feed",
"site": "https://changelog.com/gotime"
},
{
"category": "Programming",
"name": "JS Party",
"feedUrl": "https://changelog.com/jsparty/feed",
"site": "https://changelog.com/jsparty"
},
{
"category": "Programming",
"name": "Software Engineering Radio",
"feedUrl": "http://feeds.feedburner.com/se-radio",
"site": "http://www.se-radio.net/"
},
{
"category": "Programming",
"name": "Talking Machines",
"feedUrl": "https://rss.art19.com/talking-machines",
"site": "https://art19.com/shows/talking-machines"
},
{
"category": "Programming",
"name": "The Modern Web",
"feedUrl": "http://modernweb.podbean.com/feed/",
"site": "http://modernweb.podbean.com/"
},
{
"category": "Programming",
"name": "Shoptalk",
"feedUrl": "http://shoptalkshow.com/feed/podcast",
"site": "https://player.fm/series/shoptalk-19036"
},
{
"category": "Gaming",
"name": "Kinda Funny Gamescast",
"feedUrl":
"http://feeds.soundcloud.com/users/soundcloud:users:130508806/sounds.rss",
"site": "https://www.patreon.com/kindafunnygames"
},
{
"category": "Gaming",
"name": "Giant Bombcast",
"feedUrl": "https://www.giantbomb.com/podcast-xml/giant-bombcast",
"site": "https://www.giantbomb.com/podcasts/giant-bombcast/"
},
{
"category": "Gaming",
"name": "The Giant Beastcast",
"feedUrl": "https://www.giantbomb.com/podcast-xml/beastcast",
"site": "https://www.giantbomb.com/podcasts/beastcast/"
},
{
"category": "Gaming",
"name": "Three Moves Ahead",
"feedUrl": "https://www.idlethumbs.net/feeds/3ma",
"site": "https://www.idlethumbs.net/3ma"
},
{
"category": "Gaming",
"name": "Whats Good Games",
"feedUrl":
"http://feeds.backtracks.fm/feeds/whatsgoodgames/whats-good-games-a-video-game-podcast/feed.xml",
"site": "https://whatsgoodgames.com/"
},
{
"category": "Machine Learning & AI",
"name": "Data Skeptic",
"feedUrl": "https://dataskeptic.com/api/blog/rss",
"site": "https://dataskeptic.com"
},
{
"category": "Lifehacks",
"name": "Ted Talks",
"feedUrl": "https://www.ted.com/talks/rss",
"site": "https://www.ted.com"
},
{
"category": "News",
"name": "Policast",
"feedUrl": "https://feeds.publicradio.org/public_feeds/policast/npr/rss",
"site": "https://www.npr.org/podcasts/414694038/policast"
},
{
"category": "News",
"name": "The Daily",
"feedUrl": "http://rss.art19.com/the-daily",
"site": "https://www.nytimes.com/column/the-daily"
},
{
"category": "News",
"name": "BBC Global News",
"feedUrl": "https://podcasts.files.bbci.co.uk/p02nq0gn.rss",
"site": "https://www.nytimes.com/column/the-daily"
},
{
"category": "News",
"name": "PBS NewsHour",
"feedUrl": "https://www.pbs.org/newshour/feeds/rss/podcasts/show",
"site": "https://www.pbs.org/newshour"
},
{
"category": "News",
"name": "Unfilter",
"feedUrl": "http://unfilter.show/rss",
"site": "http://unfilter.show"
},
{
"category": "News",
"name": "On The Media",
"feedUrl": "http://feeds.wnyc.org/onthemedia",
"site": "https://www.wnycstudios.org/shows/otm/"
},
{
"category": "VR",
"name": "Everything VR & AR",
"feedUrl": "http://everythingvrar.libsyn.com/rss",
"site": "http://everythingvrar.libsyn.com"
},
{
"category": "VR",
"name": "New School VR Podcast",
"feedUrl":
"http://feeds.soundcloud.com/users/soundcloud:users:265033871/sounds.rss",
"site": "http://newschoolvr.com/"
},
{
"category": "VR",
"name": "Voices of VR Podcast – Designing for Virtual Reality",
"feedUrl": "http://voicesofvr.com/?feed=podcast",
"site": "http://voicesofvr.com/"
},
{
"category": "VR",
"name": "Rev VR",
"feedUrl":
"http://www.reverendkyle.com/index.php/component/podcastmanager/?format=raw&feedname=2",
"site": "http://www.revvrstudios.com/"
},
{
"category": "VR",
"name": "VRScout Report",
"feedUrl": "http://vrscout.libsyn.com/rss",
"site": "https://vrscout.com/podcast/"
},
{
"category": "Marketing",
"name": "The GaryVee Audio Experience",
"feedUrl": "http://askgaryvee.garyvee.libsynpro.com/rss",
"site": "http://www.garyvaynerchuk.com/"
},
{
"category": "Marketing",
"name": "Jocko Podcast",
"feedUrl": "http://jockopodcast.libsyn.com/rss",
"site": "http://jockopodcast.com/"
},
{
"category": "Marketing",
"name": "Starting from nothing",
"feedUrl": "http://thefoundation.libsyn.com/rss",
"site": "http://www.thefoundation.com/"
},
{
"category": "Marketing",
"name": "HBR IdeaCast",
"feedUrl": "http://feeds.harvardbusiness.org/harvardbusiness/ideacast",
"site": "http://hbrideacast.org/"
},
{
"category": "Marketing",
"name": "Business & Biceps",
"feedUrl": "http://businessandbiceps.libsyn.com/rss",
"site": "http://businessandbiceps.com/"
},
{
"category": "Marketing",
"name": "Advanced Selling Podcast",
"feedUrl": "http://billcaskey01.libsyn.com/rss",
"site": "http://www.advancedsellingpodcast.com/"
},
{
"category": "Marketing",
"name": "Marketing School",
"feedUrl": "http://mschool.growtheverywhere.libsynpro.com/rss",
"site": "http://mschool.growtheverywhere.libsynpro.com/podcast"
},
{
"category": "Marketing",
"name": "Online Marketing Strategies",
"feedUrl": "http://onlinemarketingpodcast.libsyn.com/rss",
"site": "http://onlinemarketingpodcast.libsyn.com/"
}
],
"rss": [
{
"category": "UI/UX",
"name": "A List Apart",
"feedUrl": "http://alistapart.com/main/feed",
"site": "http://alistapart.com"
},
{
"category": "UI/UX",
"name": "Smashing Magazine",
"feedUrl": "https://www.smashingmagazine.com/feed/",
"site": "https://www.smashingmagazine.com"
},
{
"category": "UI/UX",
"name": "Invision Blog",
"feedUrl": "https://www.invisionapp.com/blog/feed/",
"site": "https://www.invisionapp.com"
},
{
"category": "UI/UX",
"name": "Fast Co Design",
"feedUrl": "https://www.fastcodesign.com//latest/rss?truncated=true",
"site": "https://www.fastcodesign.com/"
},
{
"category": "UI/UX",
"name": "Yanko Design",
"feedUrl": "http://www.yankodesign.com/feed/",
"site": "http://www.yankodesign.com/"
},
{
"category": "UI/UX",
"name": "Dexigner",
"feedUrl": "http://feeds.dexigner.com/news",
"site": "http://dexigner.com"
},
{
"category": "UI/UX",
"name": "Behance",
"feedUrl": "http://feeds.feedburner.com/behance/vorr",
"site": "https://behance.net"
},
{
"category": "UI/UX",
"name": "HOW",
"feedUrl": "http://www.howdesign.com/feed/",
"site": "http://www.howdesign.com"
},
{
"category": "UI/UX",
"name": "Web Designer Depot",
"feedUrl": "http://feeds2.feedburner.com/webdesignerdepot",
"site": "https://www.webdesignerdepot.com/"
},
{
"category": "Startups & VC",
"name": "AVC",
"feedUrl": "http://feeds.feedburner.com/avc",
"site": "https://avc.com/"
},
{
"category": "Startups & VC",
"name": "Y Combinator Blog",
"feedUrl": "https://blog.ycombinator.com/feed/",
"site": "https://blog.ycombinator.com"
},
{
"category": "Startups & VC",
"name": "Alex Iskold",
"feedUrl": "https://alexiskold.net/feed/",
"site": "https://alexiskold.net"
},
{
"category": "Startups & VC",
"name": "David Cohen",
"feedUrl": "http://feeds.feedburner.com/DavidGCohen",
"site": "http://davidgcohen.com/"
},
{
"category": "Startups & VC",
"name": "VC Adventure",
"feedUrl": "https://www.sethlevine.com/feed",
"site": "https://www.sethlevine.com"
},
{
"category": "Startups & VC",
"name": "Mattermark Daily",
"feedUrl": "https://mattermark.com/category/mattermark-daily/feed/",
"site": "https://mattermark.com/category/mattermark-daily"
},
{
"category": "Startups & VC",
"name": "Feld Thoughts",
"feedUrl": "http://feeds.feedburner.com/FeldThoughts",
"site": "https://www.feld.com/"
},
{
"category": "Startups & VC",
"name": "Product Hunt Blog",
"feedUrl": "https://blog.producthunt.com/feed",
"site": "https://blog.producthunt.com/"
},
{
"category": "Startups & VC",
"name": "Both Sides of the Table",
"feedUrl": "https://bothsidesofthetable.com/feed",
"site": "https://bothsidesofthetable.com/"
},
{
"category": "Startups & VC",
"name": "Seeing Both Sides",
"feedUrl": "https://seeingbothsides.com/feed/",
"site": "https://seeingbothsides.com/"
},
{
"category": "Programming",
"name": "Lobste.rs",
"feedUrl": "https://lobste.rs/rss",
"site": "https://lobste.rs/"
},
{
"category": "Programming",
"name": "Reddit - Programming",
"feedUrl": "https://www.reddit.com/r/programming/.rss",
"site": "https://www.reddit.com/r/programming/"
},
{
"category": "Programming",
"name": "Hacker News",
"feedUrl": "https://news.ycombinator.com/rss",
"site": "https://news.ycombinator.com/"
},
{
"category": "Programming",
"name": "Joel on Software",
"feedUrl": "https://www.joelonsoftware.com/feed/",
"site": "https://www.joelonsoftware.com/"
},
{
"category": "Programming",
"name": "GitHub",
"feedUrl": "https://blog.github.com/feed.xml",
"site": "https://blog.github.com/"
},
{
"category": "Programming",
"name": "Toptal",
"feedUrl": "https://www.toptal.com/developers/blog.rss",
"site": "https://www.toptal.com/developers/blog"
},
{
"category": "Programming",
"name": "Daily WTF",
"feedUrl": "http://syndication.thedailywtf.com/TheDailyWtf",
"site": "http://thedailywtf.com/"
},
{
"category": "Gaming",
"name": "Indie Games",
"feedUrl": "http://indiegames.com/atom.xml",
"site": "http://indiegames.com"
},
{
"category": "Gaming",
"name": "IGN",
"feedUrl": "http://feeds.ign.com/ign/all",
"site": "http://ign.com/"
},
{
"category": "Gaming",
"name": "Kotaku",
"feedUrl": "https://kotaku.com/rss",
"site": "https://kotaku.com/"
},
{
"category": "Gaming",
"name": "GameInformer",
"feedUrl":
"http://www.gameinformer.com/b/features/rsscomments.aspx?WeblogPostID=8597456",
"site": "http://www.gameinformer.com"
},
{
"category": "Machine Learning & AI",
"name": "R Bloggers",
"feedUrl": "https://www.r-bloggers.com/feed/",
"site": "https://www.r-bloggers.com"
},
{
"category": "Machine Learning & AI",
"name": "KD Nuggets",
"feedUrl": "http://www.kdnuggets.com/feed",
"site": "http://www.kdnuggets.com"
},
{
"category": "Machine Learning & AI",
"name": "Kaggle",
"feedUrl": "http://blog.kaggle.com/feed",
"site": "https://www.kaggle.com/"
},
{
"category": "Machine Learning & AI",
"name": "OpenAI",
"feedUrl": "https://blog.openai.com/rss/",
"site": "https://blog.openai.com/"
},
{
"category": "News",
"name": "BBC",
"feedUrl": "http://feeds.bbci.co.uk/news/rss.xml?edition=us",
"site": "http://www.bbc.com/news"
},
{
"category": "VR",
"name": "Oculus",
"feedUrl": "http://uploadvr.com/feed/",
"site": "https://www.oculus.com/"
},
{
"category": "VR",
"name": "HTC Vive",
"feedUrl": "http://blog.vive.com/us/feed/",
"site": "http://blog.vive.com"
},
{
"category": "VR",
"name": "Road to VR",
"feedUrl": "https://www.roadtovr.com/feed/",
"site": "https://www.roadtovr.com/"
},
{
"category": "VR",
"name": "VR Scout",
"feedUrl": "https://vrscout.com/feed/",
"site": "https://vrscout.com/"
},
{
"category": "VR",
"name": "VR Focus",
"feedUrl": "https://www.vrfocus.com/feed/",
"site": "https://www.vrfocus.com"
},
{
"category": "Lifehacks",
"name": "Lifehacker",
"feedUrl": "https://lifehacker.com/rss",
"site": "https://lifehacker.com"
},
{
"category": "Lifehacks",
"name": "How to Geek",
"feedUrl": "https://feeds.howtogeek.com/HowToGeek",
"site": "https://www.howtogeek.com/"
},
{
"category": "Lifehacks",
"name": "Marc & Angle Hack Life",
"feedUrl": "http://feeds.feedburner.com/MarcAndAngel",
"site": "http://www.marcandangel.com/"
},
{
"category": "Lifehacks",
"name": "Lifehack",
"feedUrl": "https://www.lifehack.org/feed",
"site": "http://lifehack.org"
},
{
"category": "Lifehacks",
"name": "Dumb Little Man",
"feedUrl": "https://www.dumblittleman.com/feed/",
"site": "https://www.dumblittleman.com/"
},
{
"category": "Lifehacks",
"name": "Get Rich Slowly",
"feedUrl": "https://www.getrichslowly.org/feed/",
"site": "https://www.getrichslowly.org/"
},
{
"category": "Lifehacks",
"name": "Reddit - /r/lifehacks",
"feedUrl": "https://www.reddit.com/r/lifehacks/.rss",
"site": "https://www.reddit.com/r/lifehacks/"
},
{
"category": "Lifehacks",
"name": "1000 Lifehacks",
"feedUrl": "http://1000lifehacks.com/feed/",
"site": "http://1000lifehacks.com/"
},
{
"category": "News",
"name": "Reuters",
"feedUrl": "http://feeds.reuters.com/reuters/topNews",
"site": "https://www.reuters.com/"
},
{
"category": "News",
"name": "NY Times",
"feedUrl": "http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml",
"site": "https://nytimes.com"
},
{
"category": "News",
"name": "CNN - Top Stories",
"feedUrl": "http://rss.cnn.com/rss/cnn_topstories.rss",
"site": "https://cnn.com"
},
{
"category": "VR",
"name": "Road to VR",
"feedUrl": "http://roadtovr.com/feed",
"site": "https://roadtovr.com"
},
{
"category": "VR",
"name": "Upload VR",
"feedUrl": "https://uploadvr.com/feed",
"site": "https://uploadvr.com"
},
{
"category": "VR",
"name": "VR Scout",
"feedUrl": "https://vrscout.com/feed",
"site": "https://vrscout.com"
},
{
"category": "VR",
"name": "VRFocus",
"feedUrl": "https://vrfocus.com/feed",
"site": "https://vrfocus.com"
},
{
"category": "VR",
"name": "Virtual Reality Reporter",
"feedUrl": "https://virtualrealityreporter.com/feed",
"site": "https://virtualrealityreporter.com"
},
{
"category": "VR",
"name": "Reddit - Virtual Reality",
"feedUrl": "https://www.reddit.com/r/virtualreality/.rss",
"site": "https://reddit.com/r/virtualreality"
},
{
"category": "Marketing",
"name": "Hubspot Marketing",
"feedUrl": "https://blog.hubspot.com/marketing/rss.xml",
"site": "https://hubspot.com"
},
{
"category": "Marketing",
"name": "MarketingProfs Daily",
"feedUrl": "http://rss.marketingprofs.com/marketingprofs/daily",
"site": "https://marketingprofs.com"
},
{
"category": "Marketing",
"name": "Quicksprout",
"feedUrl": "http://feeds2.feedburner.com/quicksprout",
"site": "https://www.quicksprout.com/"
},
{
"category": "Marketing",
"name": "Kissmetrics Marketing Blog",
"feedUrl": "http://feeds.feedburner.com/KISSmetrics",
"site": "https://www.kissmetrics.com/"
},
{
"category": "Marketing",
"name": "Copyblogger",
"feedUrl": "http://feeds.copyblogger.com/copyblogger",
"site": "https://www.copyblogger.com/"
},
{
"category": "Marketing",
"name": "Moz",
"feedUrl": "http://feedpress.me/mozblog",
"site": "https://moz.com/"
},
{
"category": "Marketing",
"name": "Duct Tape Marketing",
"feedUrl": "https://www.ducttapemarketing.com/feed",
"site": "https://www.ducttapemarketing.com"
}
]
}
================================================
FILE: api/src/loadenv.js
================================================
import dotenv from 'dotenv';
import path from 'path';
// workaround based on https://github.com/motdotla/dotenv/issues/133
let envPath = path.resolve(__dirname, '..', '..', 'app', '.env');
console.log(`Loading .env from ${envPath}`);
dotenv.config({ path: envPath });
================================================
FILE: api/src/models/alias.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
export const AliasSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
autopopulate: {
select: ['name', 'email', 'username'],
},
},
rss: {
type: Schema.Types.ObjectId,
ref: 'RSS',
autopopulate: {
select: ['url', 'title'],
},
},
podcast: {
type: Schema.Types.ObjectId,
ref: 'Podcast',
autopopulate: {
select: ['url', 'title'],
},
},
alias: {
type: String,
trim: true,
required: true,
},
},
{ collection: 'aliases' },
);
AliasSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
AliasSchema.plugin(mongooseStringQuery);
AliasSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Alias', AliasSchema);
================================================
FILE: api/src/models/article.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
import Content from './content';
import { ParseContent } from '../parsers/content';
import { getUrl } from '../utils/urls';
import sanitize from '../utils/sanitize';
import { isBlockedURLs } from '../utils/blockedURLs';
import { EnclosureSchema } from './enclosure';
export const ArticleSchema = new Schema(
{
rss: {
type: Schema.Types.ObjectId,
ref: 'RSS',
required: true,
autopopulate: {
select: [
'title',
'url',
'feedUrl',
'favicon',
'categories',
'description',
'public',
'valid',
'publicationDate',
'lastScraped',
'images',
'featured',
],
},
index: true,
},
duplicateOf: {
type: Schema.Types.ObjectId,
ref: 'Article',
required: false,
},
url: {
type: String,
trim: true,
required: true,
index: { type: 'hashed' },
},
canonicalUrl: {
type: String,
trim: true,
},
fingerprint: {
type: String,
trim: true,
required: true,
},
guid: {
type: String,
trim: true,
},
link: {
type: String,
trim: true,
},
title: {
type: String,
trim: true,
required: true,
},
description: {
type: String,
trim: true,
// maxLength: 240,
default: '',
},
content: {
type: String,
trim: true,
default: '',
},
commentUrl: {
type: String,
trim: true,
default: '',
},
images: {
featured: {
type: String,
trim: true,
default: '',
},
banner: {
type: String,
trim: true,
default: '',
},
favicon: {
type: String,
trim: true,
default: '',
},
og: {
type: String,
trim: true,
default: '',
},
},
publicationDate: {
type: Date,
default: Date.now,
},
enclosures: [EnclosureSchema],
likes: {
type: Number,
default: 0,
},
socialScore: {
reddit: {
type: Number,
},
hackernews: {
type: Number,
},
},
valid: {
type: Boolean,
default: true,
valid: true,
},
},
{
collection: 'articles',
toJSON: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.type = 'articles';
},
},
toObject: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.type = 'articles';
},
},
},
);
ArticleSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
ArticleSchema.plugin(mongooseStringQuery);
ArticleSchema.plugin(autopopulate);
ArticleSchema.index({ rss: 1, fingerprint: 1 }, { unique: true });
ArticleSchema.index({ rss: 1, publicationDate: -1 });
ArticleSchema.index({ publicationDate: -1 });
ArticleSchema.methods.getUrl = function () {
return getUrl('article_detail', this.rss._id, this._id);
};
ArticleSchema.methods.getParsedArticle = async function () {
const url = this.url;
const content = await Content.findOne({ url });
if (content) return content;
if (isBlockedURLs(url)) {
throw new Error(`Blocked URL: ${this.url}`);
}
try {
const parsed = await ParseContent(url);
const title = parsed.title || this.title;
const excerpt = parsed.excerpt || title || this.description;
if (!title) return null;
let content = sanitize(parsed.content);
// XKCD doesn't like Mercury
if (this.url.indexOf('https://xkcd') === 0) content = this.content;
return await Content.create({
content,
title,
url,
excerpt,
image: parsed.lead_image_url || '',
publicationDate: parsed.date_published || this.publicationDate,
commentUrl: this.commentUrl,
enclosures: this.enclosures,
});
} catch (e) {
throw new Error(`Mercury call failed for ${this.url}: ${e.message}`);
}
};
module.exports = exports = mongoose.model('Article', ArticleSchema);
module.exports.ArticleSchema = ArticleSchema;
================================================
FILE: api/src/models/content.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
export const ContentSchema = new Schema(
{
url: {
type: String,
trim: true,
index: true,
required: true,
},
title: {
type: String,
trim: true,
required: true,
},
excerpt: {
type: String,
trim: true,
required: true,
},
content: {
type: String,
trim: true,
required: true,
},
image: {
type: String,
trim: true,
},
publicationDate: {
type: Date,
default: Date.now,
},
enclosures: [],
},
{ collection: 'content' },
);
ContentSchema.index({ url: 1 }, { unique: true });
ContentSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
ContentSchema.plugin(mongooseStringQuery);
module.exports = exports = mongoose.model('Content', ContentSchema);
================================================
FILE: api/src/models/enclosure.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
import { createHash } from 'crypto';
export const EnclosureSchema = new Schema({
url: {
type: String,
trim: true,
},
type: {
type: String,
trim: true,
},
length: {
type: String,
trim: true,
},
});
================================================
FILE: api/src/models/episode.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
import 'crypto';
import { createHash } from 'crypto';
import { EnclosureSchema } from './enclosure';
import Content from './content';
import { ParseContent } from '../parsers/content';
import { getUrl } from '../utils/urls';
import sanitize from '../utils/sanitize';
export const EpisodeSchema = new Schema(
{
podcast: {
type: Schema.Types.ObjectId,
ref: 'Podcast',
required: true,
autopopulate: {
select: [
'title',
'url',
'link',
'enclosure',
'feedUrl',
'image',
'categories',
'description',
'public',
'valid',
'publicationDate',
'lastScraped',
'images',
'featured',
'duplicateOf',
],
},
},
duplicateOf: {
type: Schema.Types.ObjectId,
ref: 'Episode',
required: false,
},
url: {
type: String,
trim: true,
required: true,
index: { type: 'hashed' },
},
canonicalUrl: {
type: String,
trim: true,
},
fingerprint: {
type: String,
trim: true,
required: true,
},
guid: {
type: String,
trim: true,
},
link: {
type: String,
trim: true,
index: { type: 'hashed' },
},
enclosure: {
type: String,
trim: true,
},
enclosures: [EnclosureSchema],
title: {
type: String,
trim: true,
required: true,
},
description: {
type: String,
trim: true,
// maxLength: 240,
default: '',
},
images: {
featured: {
type: String,
trim: true,
default: '',
},
banner: {
type: String,
trim: true,
default: '',
},
favicon: {
type: String,
trim: true,
default: '',
},
og: {
type: String,
trim: true,
default: '',
},
},
duration: {
type: String,
default: '',
},
publicationDate: {
type: Date,
default: Date.now,
},
likes: {
type: Number,
default: 0,
},
valid: {
type: Boolean,
default: true,
},
},
{
collection: 'episodes',
toJSON: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.type = 'episodes';
},
},
toObject: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.type = 'episodes';
},
},
},
);
EpisodeSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
EpisodeSchema.plugin(mongooseStringQuery);
EpisodeSchema.plugin(autopopulate);
EpisodeSchema.index({ podcast: 1, fingerprint: 1 }, { unique: true });
EpisodeSchema.index({ podcast: 1, publicationDate: -1 });
EpisodeSchema.index({ publicationDate: -1 });
EpisodeSchema.methods.getUrl = function () {
return getUrl('episode_detail', this.podcast._id, this._id);
};
EpisodeSchema.methods.getParsedEpisode = async function () {
const url = this.url;
const content = await Content.findOne({ url });
if (content) return content;
try {
const parsed = await ParseContent(url);
const title = parsed.title || this.title;
const excerpt = parsed.excerpt || title || this.description;
if (!title) return null;
const content = sanitize(parsed.content);
return await Content.create({
content,
title,
url,
excerpt,
image: parsed.lead_image_url || '',
publicationDate: this.publicationDate || parsed.date_published,
commentUrl: this.commentUrl,
enclosures: this.enclosures,
});
} catch (e) {
throw new Error(`Mercury call failed for ${url}: ${e.message}`);
}
};
module.exports = exports = mongoose.model('Episode', EpisodeSchema);
================================================
FILE: api/src/models/folder.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
import { getStreamClient } from '../utils/stream';
export const FolderSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
autopopulate: {
select: ['name', 'email', 'username'],
},
},
rss: [
{
type: Schema.Types.ObjectId,
ref: 'RSS',
required: true,
autopopulate: true,
},
],
podcast: [
{
type: Schema.Types.ObjectId,
ref: 'Podcast',
required: true,
autopopulate: true,
},
],
name: {
type: String,
trim: true,
required: true,
},
},
{
collection: 'folders',
toJSON: {
transform: function (doc, ret) {
ret.streamToken = getStreamClient()
.feed('folder', ret._id)
.getReadOnlyToken();
},
},
toObject: {
transform: function (doc, ret) {
ret.streamToken = getStreamClient()
.feed('folder', ret._id)
.getReadOnlyToken();
},
},
},
);
FolderSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
FolderSchema.plugin(mongooseStringQuery);
FolderSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Folder', FolderSchema);
================================================
FILE: api/src/models/follow.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
import stream from 'getstream';
import config from '../config';
import RSS from './rss';
import Podcast from './podcast';
import { getStreamClient } from '../utils/stream';
export const FollowSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
autopopulate: {
select: [
'name',
'email',
'username',
'bio',
'url',
'twitter',
'background',
'admin',
],
},
index: true,
},
followee: {
type: Schema.Types.ObjectId,
ref: 'User',
autopopulate: {
select: [
'name',
'email',
'username',
'bio',
'url',
'twitter',
'background',
],
},
index: true,
},
podcast: {
type: Schema.Types.ObjectId,
ref: 'Podcast',
autopopulate: {
select: [
'url',
'title',
'categories',
'description',
'feedUrl',
'image',
'publicationDate',
'public',
'featured',
'images',
'duplicateOf',
],
},
index: true,
},
rss: {
type: Schema.Types.ObjectId,
ref: 'RSS',
autopopulate: {
select: [
'url',
'title',
'categories',
'description',
'favicon',
'publicationDate',
'public',
'featured',
'images',
'feedUrl',
'duplicateOf',
],
},
index: true,
},
feed: {
type: String,
enum: ['rss', 'podcast', 'timeline'],
},
},
{ collection: 'follows' },
);
FollowSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
FollowSchema.plugin(mongooseStringQuery);
FollowSchema.plugin(autopopulate);
FollowSchema.index({ user: 1, rss: 1, podcast: 1 }, { unique: true });
FollowSchema.methods.removeFromStream = async function remove(follows) {
const publicationType = this.rss ? 'rss' : 'podcast';
const feedGroup = this.rss ? 'user_article' : 'user_episode';
const publicationID = this.rss ? this.rss._id : this.podcast._id;
if (!this.user) {
return [];
}
const timelineFeed = getStreamClient().feed('timeline', this.user._id);
const otherFeed = getStreamClient().feed(feedGroup, this.user._id);
const results = await Promise.all([
timelineFeed.unfollow(publicationType, publicationID),
otherFeed.unfollow(publicationType, publicationID),
]);
return results;
};
FollowSchema.statics.getOrCreateMany = async function getOrCreateMany(follows) {
// validate
for (const f of follows) {
if (f.type != 'rss' && f.type != 'podcast') {
throw new Error(`invalid follow type ${f.type}`);
}
}
// batch create the follow relationships
const followInstances = await Promise.all(
follows.map(async (f) => {
const query = { [f.type]: f.publicationID, user: f.userID };
return this.findOneAndUpdate(query, query, {
upsert: true,
new: true,
}).lean();
}),
);
// sync to stream in a batch
const feedRelationsTimeline = follows.map((f) => {
return {
source: `timeline:${f.userID}`,
target: `${f.type}:${f.publicationID}`,
};
});
const feedRelationsGroup = follows.map((f) => {
const feedGroup = f.type == 'rss' ? 'user_article' : 'user_episode';
return {
source: `${feedGroup}:${f.userID}`,
target: `${f.type}:${f.publicationID}`,
};
});
const feedRelations = feedRelationsTimeline.concat(feedRelationsGroup);
if (feedRelations.length > 0) {
await getStreamClient().followMany(feedRelations);
}
// update the counts
await Promise.all(
follows.map(async (f) => {
const followerCount = await this.count({ [f.type]: f.publicationID });
const schema = f.type == 'rss' ? RSS : Podcast;
await schema.update({ _id: f.publicationID }, { followerCount });
}),
);
return followInstances;
};
FollowSchema.statics.getOrCreate = async function getOrCreate(
followType,
userID,
publicationID,
) {
const instances = await this.getOrCreateMany([
{ type: followType, userID: userID, publicationID: publicationID },
]);
return instances[0];
};
module.exports = exports = mongoose.model('Follow', FollowSchema);
module.exports.FollowSchema = FollowSchema;
================================================
FILE: api/src/models/listen.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
export const ListenSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
autopopulate: {
select: [
'name',
'email',
'username',
'bio',
'url',
'twitter',
'preferences',
'background',
'admin',
],
},
},
episode: {
type: Schema.Types.ObjectId,
ref: 'Episode',
autopopulate: {
select: [
'parent',
'url',
'title',
'description',
'image',
'publictionDate',
],
},
required: true,
},
duration: {
type: Number,
default: 0,
required: true,
},
},
{ collection: 'listens' },
);
ListenSchema.index({ user: 1, episode: 1 }, { unique: true });
ListenSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
ListenSchema.plugin(mongooseStringQuery);
ListenSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Listen', ListenSchema);
================================================
FILE: api/src/models/note.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
export const NoteSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
},
episode: {
type: Schema.Types.ObjectId,
ref: 'Episode',
autopopulate: { select: ['title', 'podcast'], maxDepth: 1 },
},
article: {
type: Schema.Types.ObjectId,
ref: 'Article',
autopopulate: { select: ['title', 'rss'], maxDepth: 1 },
},
start: {
type: Number,
required: true,
},
end: {
type: Number,
required: true,
},
// text===null? it's a highlight : it's a note
text: {
type: String,
trim: true,
},
},
{
collection: 'notes',
},
);
NoteSchema.index({ user: 1, article: 1 }, { index: true });
NoteSchema.index({ user: 1, episode: 1 }, { index: true });
NoteSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
NoteSchema.plugin(mongooseStringQuery);
NoteSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Note', NoteSchema);
================================================
FILE: api/src/models/pin.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
export const PinSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
autopopulate: {
select: [
'name',
'email',
'username',
'bio',
'url',
'twitter',
'background',
'admin',
],
},
},
article: {
type: Schema.Types.ObjectId,
ref: 'Article',
autopopulate: {
select: [
'commentUrl',
'parent',
'url',
'title',
'description',
'images',
'publicationDate',
'enclosures',
],
},
},
episode: {
type: Schema.Types.ObjectId,
ref: 'Episode',
autopopulate: {
select: [
'parent',
'url',
'title',
'description',
'images',
'publicationDate',
'link',
],
},
},
url: {
type: String,
trim: true,
},
},
{ collection: 'pins' },
);
PinSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
PinSchema.plugin(mongooseStringQuery);
PinSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Pin', PinSchema);
================================================
FILE: api/src/models/playlist.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import autopopulate from 'mongoose-autopopulate';
export const PlaylistSchema = new Schema(
{
user: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
autopopulate: {
select: [
'name',
'email',
'username',
'bio',
'url',
'twitter',
'background',
'admin',
],
},
},
name: {
type: String,
trim: true,
required: true,
},
episodes: [
{
type: Schema.Types.ObjectId,
ref: 'Episode',
required: true,
autopopulate: true,
},
],
likes: {
type: Number,
default: 0,
},
},
{ collection: 'playlists' },
);
PlaylistSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
PlaylistSchema.plugin(mongooseStringQuery);
PlaylistSchema.plugin(autopopulate);
module.exports = exports = mongoose.model('Playlist', PlaylistSchema);
================================================
FILE: api/src/models/podcast.js
================================================
import mongoose, { Schema } from 'mongoose';
import timestamps from 'mongoose-timestamp';
import mongooseStringQuery from 'mongoose-string-query';
import { getStreamClient } from '../utils/stream';
import { getUrl } from '../utils/urls';
export const PodcastSchema = new Schema(
{
duplicateOf: {
type: Schema.Types.ObjectId,
ref: 'Podcast',
required: false,
},
url: {
type: String,
trim: true,
},
canonicalUrl: {
type: String,
trim: true,
},
feedUrl: {
type: String,
trim: true,
index: true,
unique: true,
required: true,
},
feedUrls: [String],
fingerprint: {
type: String,
trim: true,
},
title: {
type: String,
trim: true,
required: true,
},
description: {
type: String,
trim: true,
default: '',
},
summary: {
type: String,
trim: true,
default: '',
},
categories: {
type: String,
trim: true,
default: '',
},
featured: {
type: Boolean,
default: false,
},
images: {
featured: {
type: String,
trim: true,
default: '',
},
banner: {
type: String,
trim: true,
default: '',
},
favicon: {
type: String,
trim: true,
default: '',
},
og: {
type: String,
trim: true,
default: '',
},
},
public: {
type: Boolean,
default: true,
},
publicationDate: {
type: Date,
default: Date.now,
},
valid: {
type: Boolean,
default: true,
},
lastScraped: {
type: Date,
default: Date.now,
},
interest: {
type: String,
default: '',
index: true,
},
language: {
type: String,
default: '',
},
followerCount: {
type: Number,
default: 0,
},
// @deprecated: considering the huge collection size the count op is really slow
postCount: {
type: Number,
default: 0,
},
consecutiveScrapeFailures: {
type: Number,
default: 0,
},
guidStability: {
type: String,
enum: ['STABLE', 'UNSTABLE', 'UNCHECKED'],
default: 'UNCHECKED',
},
},
{
collection: 'podcasts',
toJSON: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.streamToken = getStreamClient()
.feed('podcast', ret._id)
.getReadOnlyToken();
},
},
toObject: {
transform: function (doc, ret) {
// Frontend breaks if images is null, should be {} instead
if (!ret.images) {
ret.images = {};
}
ret.images.favicon = ret.images.favicon || '';
ret.images.og = ret.images.og || '';
ret.streamToken = getStreamClient()
.feed('podcast', ret._id)
.getReadOnlyToken();
},
},
},
);
PodcastSchema.index({ featured: 1 }, { partialFilterExpression: { featured: true } });
PodcastSchema.index({ valid: 1, followerCount: -1 });
PodcastSchema.plugin(timestamps, {
createdAt: { index: true },
updatedAt: { index: true },
});
PodcastSchema.plugin(mongooseStringQuery);
PodcastSchema.statics.incrScrapeFailures = async function (id) {
await this.findOneAndUpdate(
{ _id: id },
{ $inc: { consecutiveScrapeFailures: 1 } },
).exec();
};
PodcastSchema.statics.resetScrapeFailures = async function (id) {
await this.findOneAndUpdate(
{ _id: id },
{ $set: { consecutiveScrapeFailures: 0 } },
).exec();
};
PodcastSchema.methods.searchDocument = function () {
return {
_id: this._id,
objectID: this._id,
categories: 'Podcast',
description: this
gitextract_ss1x8z9z/
├── .eslintrc.js
├── .gitattributes
├── .github/
│ ├── ISSUE_TEMPLATE.md
│ ├── PULL_REQUEST_TEMPLATE.md
│ └── stale.yml
├── .gitignore
├── .prettierrc
├── .travis.yml
├── Dockerfile
├── LICENSE
├── README.md
├── RSS.md
├── STYLE.md
├── api/
│ ├── Dockerfile
│ ├── build.sh
│ ├── config.yaml
│ ├── docker-compose.yml
│ ├── ecosystem.dev.config.js
│ ├── now.json
│ ├── package.json
│ ├── scripts/
│ │ ├── docker-build.sh
│ │ ├── docker-compose-aws.sh
│ │ ├── docker-compose.sh
│ │ └── make-build.sh
│ ├── setup-tests.js
│ ├── src/
│ │ ├── .babelrc
│ │ ├── .prettierignore
│ │ ├── asyncTasks.js
│ │ ├── commands/
│ │ │ ├── _debug-feed.js
│ │ │ ├── cleanup-follows.js
│ │ │ ├── denormalize-follows.js
│ │ │ ├── denormalize-pins.js
│ │ │ ├── email.js
│ │ │ ├── load-featured-feeds.js
│ │ │ ├── rescrape-favicon.js
│ │ │ ├── rescrape-og.js
│ │ │ ├── reset-parsing-state.js
│ │ │ ├── resync-follows.js
│ │ │ ├── winds-article.js
│ │ │ ├── winds-discover.js
│ │ │ ├── winds-merge.js
│ │ │ ├── winds-og.js
│ │ │ ├── winds-podcast.js
│ │ │ ├── winds-rebuild-search.js
│ │ │ ├── winds-rehash.js
│ │ │ ├── winds-rss.js
│ │ │ ├── winds-truncate-rss-feed.js
│ │ │ └── winds.js
│ │ ├── config/
│ │ │ ├── dev.js
│ │ │ ├── index.js
│ │ │ ├── prod.js
│ │ │ └── test.js
│ │ ├── controllers/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── default.js
│ │ │ ├── email.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── feed.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── fixtures/
│ │ │ └── featured.json
│ │ ├── loadenv.js
│ │ ├── models/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── content.js
│ │ │ ├── enclosure.js
│ │ │ ├── episode.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── parsers/
│ │ │ ├── content.js
│ │ │ ├── detect-language.js
│ │ │ ├── detect-type.js
│ │ │ ├── discovery.js
│ │ │ ├── feed.js
│ │ │ └── og.js
│ │ ├── routes/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── email.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── index.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── server.js
│ │ ├── utils/
│ │ │ ├── analytics.js
│ │ │ ├── basicAuth.js
│ │ │ ├── blockedURLs.js
│ │ │ ├── collections.js
│ │ │ ├── controllers.js
│ │ │ ├── db/
│ │ │ │ └── index.js
│ │ │ ├── email/
│ │ │ │ ├── context.js
│ │ │ │ ├── send.js
│ │ │ │ └── templates/
│ │ │ │ ├── daily.ejs
│ │ │ │ ├── followee.ejs
│ │ │ │ ├── password.ejs
│ │ │ │ ├── weekly.ejs
│ │ │ │ └── welcome.ejs
│ │ │ ├── errors.js
│ │ │ ├── logger/
│ │ │ │ ├── index.js
│ │ │ │ └── sentry.js
│ │ │ ├── merge.js
│ │ │ ├── personalization/
│ │ │ │ └── index.js
│ │ │ ├── queue.js
│ │ │ ├── random.js
│ │ │ ├── rate-limiter.js
│ │ │ ├── sanitize.js
│ │ │ ├── search/
│ │ │ │ └── index.js
│ │ │ ├── social.js
│ │ │ ├── statsd.js
│ │ │ ├── stream.js
│ │ │ ├── upsert.js
│ │ │ ├── urls.js
│ │ │ ├── validation.js
│ │ │ └── watchdog.js
│ │ └── workers/
│ │ ├── conductor.js
│ │ ├── og.js
│ │ ├── podcast.js
│ │ ├── rss.js
│ │ ├── social.js
│ │ ├── stream.js
│ │ ├── winds-hackernews.js
│ │ └── winds-queue-state-monitor.js
│ ├── test/
│ │ ├── controllers/
│ │ │ ├── alias.js
│ │ │ ├── article.js
│ │ │ ├── auth.js
│ │ │ ├── episode.js
│ │ │ ├── featured.js
│ │ │ ├── feed.js
│ │ │ ├── folder.js
│ │ │ ├── follow.js
│ │ │ ├── health.js
│ │ │ ├── listen.js
│ │ │ ├── note.js
│ │ │ ├── opml.js
│ │ │ ├── pin.js
│ │ │ ├── playlist.js
│ │ │ ├── podcast.js
│ │ │ ├── rss.js
│ │ │ ├── tag.js
│ │ │ └── user.js
│ │ ├── data/
│ │ │ ├── 404.opml
│ │ │ ├── discovery/
│ │ │ │ ├── case.html
│ │ │ │ ├── fail.xml
│ │ │ │ ├── index.html
│ │ │ │ ├── nofavicon.html
│ │ │ │ ├── nourl.xml
│ │ │ │ └── rss.xml
│ │ │ ├── feed/
│ │ │ │ ├── 90.cx
│ │ │ │ ├── a16z
│ │ │ │ ├── api.prprpr.me
│ │ │ │ ├── apublica.org
│ │ │ │ ├── audiworld.com
│ │ │ │ ├── boingboing
│ │ │ │ ├── bookshadow.com
│ │ │ │ ├── dingxiaoyun555.blog.163.com
│ │ │ │ ├── django
│ │ │ │ ├── douban.com
│ │ │ │ ├── empty
│ │ │ │ ├── geektopia.es
│ │ │ │ ├── habr
│ │ │ │ ├── hackernews
│ │ │ │ ├── hackernoon-daily-dev
│ │ │ │ ├── kaiak.tw
│ │ │ │ ├── kottke
│ │ │ │ ├── lemonde
│ │ │ │ ├── lobsters
│ │ │ │ ├── lowendbox.com
│ │ │ │ ├── malformed-hackernews
│ │ │ │ ├── maxwell-land-surveying.com
│ │ │ │ ├── medium-technology
│ │ │ │ ├── perezhilton
│ │ │ │ ├── reddit-r-programming
│ │ │ │ ├── rss.cnki.net
│ │ │ │ ├── ruby-on-rails
│ │ │ │ ├── seattle.craigslist.org
│ │ │ │ ├── shanzhuoboshi.com
│ │ │ │ ├── sospc.name
│ │ │ │ ├── straitstimes.com
│ │ │ │ ├── strava
│ │ │ │ ├── stream
│ │ │ │ ├── techcrunch
│ │ │ │ ├── tejiendoelmundo.wordpress.com
│ │ │ │ ├── thewildeternal.com
│ │ │ │ ├── tmz
│ │ │ │ ├── torrentedigital.com
│ │ │ │ ├── totoyao.wordpress.com
│ │ │ │ ├── treehugger-latest
│ │ │ │ ├── ttt.tt
│ │ │ │ ├── xda-developers.com
│ │ │ │ └── zhukun.net
│ │ │ ├── not-a-url.opml
│ │ │ ├── og/
│ │ │ │ ├── bildblog.html
│ │ │ │ ├── google.html
│ │ │ │ ├── kotaku.html
│ │ │ │ ├── techcrunch.html
│ │ │ │ ├── techcrunch_broken.html
│ │ │ │ └── techcrunch_instagram.html
│ │ │ ├── podcast-feed/
│ │ │ │ ├── a16z
│ │ │ │ ├── atlantamonster
│ │ │ │ ├── buffering-the-vampire-slayer
│ │ │ │ ├── design-details
│ │ │ │ ├── giant-bombcast
│ │ │ │ ├── making-obama
│ │ │ │ ├── nancy
│ │ │ │ ├── serial
│ │ │ │ ├── still-processing
│ │ │ │ └── thehabitat
│ │ │ └── test.xml
│ │ ├── fixtures/
│ │ │ ├── aliases.json
│ │ │ ├── articles.json
│ │ │ ├── featured.json
│ │ │ ├── folders.json
│ │ │ ├── follows.json
│ │ │ ├── initial-data.json
│ │ │ ├── liked-shares.json
│ │ │ ├── listens.json
│ │ │ ├── merge-data.json
│ │ │ ├── notes.json
│ │ │ ├── opml.json
│ │ │ ├── pins.json
│ │ │ ├── playlists.json
│ │ │ ├── tags.json
│ │ │ ├── unstable-guid.json
│ │ │ ├── user.json
│ │ │ └── user_model.json
│ │ ├── models/
│ │ │ └── user.js
│ │ ├── parsers/
│ │ │ ├── content.js
│ │ │ ├── discovery.js
│ │ │ ├── feed.js
│ │ │ ├── language.js
│ │ │ └── og.js
│ │ ├── server.js
│ │ ├── utilities/
│ │ │ ├── collections.js
│ │ │ ├── email.js
│ │ │ ├── merge.js
│ │ │ ├── random.js
│ │ │ ├── rate-limiter.js
│ │ │ ├── social.js
│ │ │ ├── upsert.js
│ │ │ ├── url.js
│ │ │ └── validation.js
│ │ ├── utils.js
│ │ └── workers/
│ │ ├── conductor.js
│ │ ├── og.js
│ │ ├── podcast.js
│ │ ├── rss.js
│ │ ├── social.js
│ │ └── stream.js
│ └── test-entry.js
├── app/
│ ├── electron-builder.yaml
│ ├── package.json
│ ├── public/
│ │ ├── actions.js
│ │ ├── electron.js
│ │ ├── entitlements.mac.plist
│ │ ├── entitlements.mas.plist
│ │ ├── favicons/
│ │ │ └── icon.icns
│ │ ├── index.html
│ │ ├── info.plist
│ │ ├── manifest.json
│ │ └── preload.js
│ ├── src/
│ │ ├── App.js
│ │ ├── AppRouter.js
│ │ ├── AuthedRoute.js
│ │ ├── UnauthedRoute.js
│ │ ├── api/
│ │ │ ├── folderAPI.js
│ │ │ ├── index.js
│ │ │ ├── noteAPI.js
│ │ │ └── tagAPI.js
│ │ ├── components/
│ │ │ ├── AddOPMLModal.js
│ │ │ ├── AddPodcastModal.js
│ │ │ ├── AddRSSModal.js
│ │ │ ├── AliasModal.js
│ │ │ ├── AllArticlesList.js
│ │ │ ├── AllEpisodesList.js
│ │ │ ├── ArticleListItem.js
│ │ │ ├── Avatar/
│ │ │ │ └── index.js
│ │ │ ├── BookmarkPanel.js
│ │ │ ├── Drawer.js
│ │ │ ├── EpisodeListItem.js
│ │ │ ├── FeaturedItems.js
│ │ │ ├── FeedHeader.js
│ │ │ ├── FeedListItem.js
│ │ │ ├── Folder/
│ │ │ │ ├── DeleteModal.js
│ │ │ │ ├── FeedToFolderModal.js
│ │ │ │ ├── Folder.js
│ │ │ │ ├── FolderFeeds.js
│ │ │ │ ├── FolderPanel.js
│ │ │ │ ├── FolderPopover.js
│ │ │ │ ├── IntroFolders.js
│ │ │ │ ├── NewFolderModal.js
│ │ │ │ ├── RenameModal.js
│ │ │ │ └── SearchFeed.js
│ │ │ ├── Header.js
│ │ │ ├── HtmlRender.js
│ │ │ ├── Loader.js
│ │ │ ├── MediaCard.js
│ │ │ ├── Notes/
│ │ │ │ ├── HighlightMenu.js
│ │ │ │ ├── NoteInput.js
│ │ │ │ └── RecentNotesPanel.js
│ │ │ ├── Panel.js
│ │ │ ├── Player.js
│ │ │ ├── PodcastEpisode.js
│ │ │ ├── PodcastEpisodesView.js
│ │ │ ├── PodcastPanels/
│ │ │ │ ├── PodcastList.js
│ │ │ │ ├── RecentEpisodesPanel.js
│ │ │ │ ├── SuggestedPodcasts.js
│ │ │ │ └── index.js
│ │ │ ├── RSSArticle.js
│ │ │ ├── RSSArticleList.js
│ │ │ ├── RSSPanels/
│ │ │ │ ├── RecentArticlesPanel.js
│ │ │ │ ├── RssFeedList.js
│ │ │ │ ├── SuggestedFeeds.js
│ │ │ │ └── index.js
│ │ │ ├── SearchBar.js
│ │ │ ├── SimpleProgressBar.js
│ │ │ ├── Tabs.js
│ │ │ ├── Tag/
│ │ │ │ ├── DeleteModal.js
│ │ │ │ ├── RenameModal.js
│ │ │ │ ├── Tag.js
│ │ │ │ ├── TagFeeds.js
│ │ │ │ ├── TagPanel.js
│ │ │ │ └── TagView.js
│ │ │ ├── TimeAgo/
│ │ │ │ └── index.js
│ │ │ └── UserProfileSettingsDrawer.js
│ │ ├── config.js
│ │ ├── index.js
│ │ ├── reducers.js
│ │ ├── serviceWorker.js
│ │ ├── static-data/
│ │ │ └── onboarding-topics.js
│ │ ├── styles/
│ │ │ ├── base/
│ │ │ │ ├── _colors.scss
│ │ │ │ ├── _normalize.scss
│ │ │ │ ├── _typography.scss
│ │ │ │ └── _vars.scss
│ │ │ ├── components/
│ │ │ │ ├── _click-catcher.scss
│ │ │ │ ├── _columns.scss
│ │ │ │ ├── _comment-input-box.scss
│ │ │ │ ├── _comment-section.scss
│ │ │ │ ├── _content.scss
│ │ │ │ ├── _episode-info-popover.scss
│ │ │ │ ├── _feed-header.scss
│ │ │ │ ├── _feed-list-item.scss
│ │ │ │ ├── _follow-popover.scss
│ │ │ │ ├── _github.scss
│ │ │ │ ├── _hero-card.scss
│ │ │ │ ├── _item-info.scss
│ │ │ │ ├── _list.scss
│ │ │ │ ├── _loader.scss
│ │ │ │ ├── _media-card.scss
│ │ │ │ ├── _panel-element.scss
│ │ │ │ ├── _playlist-card.scss
│ │ │ │ ├── _popover-panel.scss
│ │ │ │ ├── _shows-grid.scss
│ │ │ │ ├── _simple-progress-bar.scss
│ │ │ │ └── _tiny-list.scss
│ │ │ ├── elements/
│ │ │ │ ├── _buttons.scss
│ │ │ │ ├── _drawer.scss
│ │ │ │ ├── _forms.scss
│ │ │ │ ├── _modals.scss
│ │ │ │ ├── _popovers.scss
│ │ │ │ └── _tabs.scss
│ │ │ ├── framework/
│ │ │ │ ├── _app.scss
│ │ │ │ ├── _player.scss
│ │ │ │ ├── auth-views/
│ │ │ │ │ ├── _create.scss
│ │ │ │ │ ├── _forgot-password.scss
│ │ │ │ │ ├── _login.scss
│ │ │ │ │ ├── _reset-password.scss
│ │ │ │ │ └── _shared.scss
│ │ │ │ └── dashboard/
│ │ │ │ └── _header.scss
│ │ │ ├── global.scss
│ │ │ ├── modules/
│ │ │ │ ├── _activity-feed.scss
│ │ │ │ ├── _add-content.scss
│ │ │ │ ├── _download.scss
│ │ │ │ ├── _featured-items-section.scss
│ │ │ │ ├── _follow-suggestions.scss
│ │ │ │ ├── _my-playlists.scss
│ │ │ │ ├── _notification-feed.scss
│ │ │ │ ├── _podcast-suggestions.scss
│ │ │ │ ├── _reshare-modal.scss
│ │ │ │ ├── _rss-panels.scss
│ │ │ │ ├── _search-results.scss
│ │ │ │ ├── _share.scss
│ │ │ │ ├── _social-icons.scss
│ │ │ │ └── _user-settings-drawer.scss
│ │ │ └── views/
│ │ │ ├── _404.scss
│ │ │ ├── _admin.scss
│ │ │ ├── _dashboard.scss
│ │ │ ├── _folder.scss
│ │ │ ├── _grid.scss
│ │ │ ├── _note.scss
│ │ │ ├── _onboarding.scss
│ │ │ ├── _playlist.scss
│ │ │ ├── _podcast.scss
│ │ │ ├── _profile.scss
│ │ │ └── _rss.scss
│ │ ├── util/
│ │ │ ├── feeds.js
│ │ │ ├── fetch/
│ │ │ │ └── index.js
│ │ │ ├── getFeedActivities.js
│ │ │ ├── getPlaceholderImageURL.js
│ │ │ ├── pins.js
│ │ │ └── social.js
│ │ └── views/
│ │ ├── 404View.js
│ │ ├── AdminView.js
│ │ ├── Dashboard.js
│ │ ├── FoldersView.js
│ │ ├── PodcastsView.js
│ │ ├── RSSFeedsView.js
│ │ └── auth-views/
│ │ ├── Create.js
│ │ ├── ForgotPassword.js
│ │ ├── Login.js
│ │ ├── ResetPassword.js
│ │ └── index.js
│ └── stylelint.config.js
├── process_dev.json
└── scripts/
└── docker-cleanup.sh
SYMBOL INDEX (466 symbols across 136 files)
FILE: api/setup-tests.js
function wrapMocha (line 28) | function wrapMocha(onPrepare, onUnprepare) {
FILE: api/src/asyncTasks.js
function makeMetricKey (line 36) | function makeMetricKey(queue, event) {
function trackQueueSize (line 40) | async function trackQueueSize(statsd, queue) {
function AddQueueTracking (line 46) | function AddQueueTracking(queue) {
function ProcessRssQueue (line 105) | function ProcessRssQueue() {
function ProcessOgQueue (line 110) | function ProcessOgQueue() {
function ProcessPodcastQueue (line 115) | function ProcessPodcastQueue() {
function ProcessStreamQueue (line 120) | function ProcessStreamQueue() {
function ProcessSocialQueue (line 125) | function ProcessSocialQueue() {
function ShutDownRssQueue (line 130) | function ShutDownRssQueue() {
function ShutDownPodcastQueue (line 135) | function ShutDownPodcastQueue() {
function ShutDownOgQueue (line 140) | function ShutDownOgQueue() {
function ShutDownSocialQueue (line 145) | function ShutDownSocialQueue() {
function ShutDownStreamQueue (line 150) | function ShutDownStreamQueue() {
FILE: api/src/commands/_debug-feed.js
function debugFeed (line 13) | async function debugFeed(feedType, feedUrls) {
FILE: api/src/commands/cleanup-follows.js
function main (line 18) | async function main() {
FILE: api/src/commands/denormalize-follows.js
function main (line 15) | async function main() {
FILE: api/src/commands/denormalize-pins.js
function main (line 17) | async function main() {
FILE: api/src/commands/email.js
function main (line 23) | async function main() {
FILE: api/src/commands/load-featured-feeds.js
function main (line 37) | function main() {
FILE: api/src/commands/rescrape-favicon.js
function rescrapeFavicon (line 16) | async function rescrapeFavicon(publicationType, instance) {
function main (line 42) | async function main() {
FILE: api/src/commands/rescrape-og.js
function partitionBy (line 15) | function partitionBy(collection, selector) {
function main (line 35) | async function main() {
FILE: api/src/commands/reset-parsing-state.js
function main (line 12) | async function main() {
FILE: api/src/commands/resync-follows.js
function main (line 16) | async function main() {
FILE: api/src/commands/winds-article.js
function main (line 15) | async function main() {
FILE: api/src/commands/winds-discover.js
function main (line 17) | async function main() {
FILE: api/src/commands/winds-merge.js
function sleep (line 11) | function sleep(time) {
function estimateSize (line 22) | function estimateSize(content) {
function main (line 32) | async function main() {
FILE: api/src/commands/winds-og.js
function main (line 20) | async function main() {
FILE: api/src/commands/winds-rebuild-search.js
function main (line 9) | async function main() {
function loadModel (line 16) | async function loadModel(Model) {
FILE: api/src/commands/winds-rehash.js
function main (line 8) | async function main() {
function rehashModel (line 13) | async function rehashModel(Model) {
FILE: api/src/commands/winds-truncate-rss-feed.js
function main (line 12) | async function main() {
FILE: api/src/commands/winds.js
function main (line 25) | function main() {
FILE: api/src/controllers/auth.js
function getInterestMap (line 23) | async function getInterestMap() {
FILE: api/src/controllers/email.js
function createEmail (line 17) | async function createEmail(type, user) {
FILE: api/src/controllers/feed.js
function getContentFeed (line 9) | async function getContentFeed(req, res, type, model) {
FILE: api/src/controllers/folder.js
function checkRssPodcast (line 154) | async function checkRssPodcast(rssIDs, podcastIDs) {
function streamFollow (line 173) | async function streamFollow(folderId, type, feedId) {
function streamUnfollow (line 178) | async function streamUnfollow(folderId, type, feedId) {
function streamFollowMany (line 183) | async function streamFollowMany(folder) {
function streamUnfollowMany (line 188) | async function streamUnfollowMany(folder) {
function generateRels (line 193) | function generateRels(folder) {
FILE: api/src/controllers/opml.js
function partitionBy (line 59) | function partitionBy(collection, selector) {
function identifyFeedType (line 79) | async function identifyFeedType(feed) {
function getOrCreateManyPublications (line 110) | async function getOrCreateManyPublications(feeds) {
FILE: api/src/controllers/playlist.js
function listFilter (line 6) | async function listFilter(req, res) {
FILE: api/src/parsers/content.js
function ParseContent (line 3) | async function ParseContent(url) {
FILE: api/src/parsers/detect-language.js
function DetectLanguage (line 5) | async function DetectLanguage(feedURL) {
function DetectLangFromStream (line 12) | async function DetectLangFromStream(feedStream) {
FILE: api/src/parsers/detect-type.js
function IsPodcastStream (line 4) | async function IsPodcastStream(feedStream) {
function IsPodcastURL (line 18) | async function IsPodcastURL(feedURL) {
FILE: api/src/parsers/discovery.js
function readRequestBody (line 37) | function readRequestBody(stream, url) {
function discoverRSS (line 97) | async function discoverRSS(uri) {
function discoverFromHTML (line 123) | function discoverFromHTML(body) {
function isRelativeUrl (line 181) | function isRelativeUrl(str) {
function getFaviconUrl (line 185) | function getFaviconUrl(uri) {
function fixData (line 192) | async function fixData(res, uri) {
function discoverFromFeed (line 245) | function discoverFromFeed(body) {
FILE: api/src/parsers/feed.js
function sanitize (line 32) | function sanitize(dirty) {
function ParsePodcast (line 39) | async function ParsePodcast(podcastUrl, guidStability, limit = 1000) {
function ParseFeed (line 52) | async function ParseFeed(feedURL, guidStability, limit = 1000) {
function ComputeHash (line 65) | function ComputeHash(post) {
function ComputePublicationHash (line 73) | function ComputePublicationHash(posts, limit = 20) {
function CreateFingerPrints (line 85) | function CreateFingerPrints(posts, guidStability) {
function ParsePodcastPosts (line 142) | function ParsePodcastPosts(domain, posts, guidStability, limit = 1000) {
function ReadURL (line 200) | function ReadURL(url) {
function sleep (line 218) | function sleep(time) {
function checkHeaders (line 225) | function checkHeaders(stream, url, checkContenType = false) {
function ReadPageURL (line 306) | async function ReadPageURL(url, retries = 2, backoffDelay = 100) {
function ReadFeedURL (line 325) | async function ReadFeedURL(feedURL, retries = 2, backoffDelay = 100) {
function ReadFeedStream (line 344) | function ReadFeedStream(feedStream) {
function ParseFeedPosts (line 380) | function ParseFeedPosts(domain, posts, guidStability, limit = 1000) {
function checkGuidStability (line 489) | function checkGuidStability(original, control) {
FILE: api/src/parsers/og.js
function ParseOG (line 7) | async function ParseOG(pageURL) {
function IsValidOGUrl (line 12) | function IsValidOGUrl(url) {
function parseImage (line 29) | function parseImage(html) {
function parseCanonicalUrl (line 47) | function parseCanonicalUrl(html) {
function ParseOGStream (line 74) | function ParseOGStream(pageStream, pageURL) {
FILE: api/src/utils/analytics.js
function getAnalyticsClient (line 9) | function getAnalyticsClient() {
function trackEngagement (line 29) | async function trackEngagement(user, engagement) {
FILE: api/src/utils/collections.js
function upsertCollections (line 12) | async function upsertCollections(type, content) {
function estimateSize (line 31) | function estimateSize(content) {
function sendFeedToCollections (line 41) | async function sendFeedToCollections(type, feed, content) {
FILE: api/src/utils/controllers.js
function wrapAsync (line 1) | function wrapAsync(fn) {
FILE: api/src/utils/email/context.js
function dailyContextGlobal (line 13) | async function dailyContextGlobal() {
function weeklyContextGlobal (line 18) | async function weeklyContextGlobal() {
function dailyContextUser (line 30) | async function dailyContextUser(user) {
function getRedirectUrl (line 35) | function getRedirectUrl(url, id, userID, position) {
function weeklyContextUser (line 47) | async function weeklyContextUser(user) {
FILE: api/src/utils/email/send.js
function CreateDailyEmail (line 10) | function CreateDailyEmail(data) {
function CreateWeeklyEmail (line 26) | function CreateWeeklyEmail(data) {
function SendDailyEmail (line 42) | async function SendDailyEmail(data) {
function SendWeeklyEmail (line 48) | async function SendWeeklyEmail(data) {
function SendWelcomeEmail (line 54) | async function SendWelcomeEmail(data) {
function SendPasswordResetEmail (line 68) | async function SendPasswordResetEmail(data) {
function SendEmail (line 85) | async function SendEmail(obj) {
FILE: api/src/utils/errors.js
function sendSourceMaps (line 18) | function sendSourceMaps(data) {
function captureError (line 48) | function captureError(err, msg) {
FILE: api/src/utils/logger/index.js
function isError (line 10) | function isError(e) {
FILE: api/src/utils/logger/sentry.js
class SentryWinstonTransport (line 62) | class SentryWinstonTransport extends Transport {
method constructor (line 63) | constructor(options) {
method log (line 85) | async log(info, done) {
function createSentryTransport (line 100) | function createSentryTransport(ravenInstance) {
FILE: api/src/utils/merge.js
function mergeFollows (line 11) | async function mergeFollows(lhsID, rhsID, type) {
function mergeArticlesAndPins (line 38) | async function mergeArticlesAndPins(lhsID, rhsID, type) {
function mergeFeedUrls (line 90) | async function mergeFeedUrls(lhsID, rhsID, type) {
function mergeFeeds (line 107) | async function mergeFeeds(lhsID, rhsID, type) {
FILE: api/src/utils/personalization/index.js
function createGlobalToken (line 11) | function createGlobalToken() {
function getRecommendations (line 25) | async function getRecommendations(userID, type, limit) {
function getRSSRecommendations (line 44) | async function getRSSRecommendations(userID, limit = 20) {
function getPodcastRecommendations (line 58) | async function getPodcastRecommendations(userID, limit = 20) {
function getEpisodeRecommendations (line 72) | async function getEpisodeRecommendations(userID, limit = 20) {
function getArticleRecommendations (line 86) | async function getArticleRecommendations(userID, limit = 20) {
FILE: api/src/utils/queue.js
constant TTL (line 7) | const TTL = 3600;
function tryAddToQueueFlagSet (line 60) | async function tryAddToQueueFlagSet(queueName, suffix, id, ttl = TTL) {
function removeFromQueueFlagSet (line 67) | async function removeFromQueueFlagSet(queueName, suffix, id) {
function getQueueFlagSetMembers (line 73) | async function getQueueFlagSetMembers(queueName) {
function tryCreateQueueFlag (line 78) | async function tryCreateQueueFlag(queueName, suffix, id, ttl = TTL) {
function removeQueueFlag (line 84) | async function removeQueueFlag(queueName, suffix, id) {
FILE: api/src/utils/random.js
function weightedRandom (line 3) | function weightedRandom(n = 6) {
FILE: api/src/utils/rate-limiter.js
function sleep (line 35) | function sleep(time) {
function tick (line 39) | async function tick(userID, requestsPerDay = 3000) {
function reset (line 52) | async function reset(userID) {
FILE: api/src/utils/social.js
function extractRedditPostID (line 7) | function extractRedditPostID(article) {
function extractHackernewsPostID (line 27) | function extractHackernewsPostID(article) {
function refreshAccessToken (line 38) | async function refreshAccessToken() {
function sleep (line 63) | function sleep(time) {
function tryRedditAPI (line 67) | async function tryRedditAPI(path, retries = 2, backoffDelay = 30) {
function tryHackernewsAPI (line 99) | async function tryHackernewsAPI(path, retries = 2, backoffDelay = 30) {
function tryHackernewsSearch (line 115) | async function tryHackernewsSearch(query, retries = 2, backoffDelay = 30) {
function redditPost (line 138) | async function redditPost(article) {
function hackernewsPost (line 144) | async function hackernewsPost(article) {
function redditScore (line 150) | async function redditScore(postID) {
function hackernewsScore (line 155) | async function hackernewsScore(postID) {
function fetchSocialScore (line 169) | async function fetchSocialScore(article) {
FILE: api/src/utils/statsd.js
function getStatsDClient (line 7) | function getStatsDClient() {
function timeIt (line 19) | async function timeIt(name, fn) {
FILE: api/src/utils/stream.js
function getStreamClient (line 6) | function getStreamClient() {
FILE: api/src/utils/upsert.js
function upsertManyPosts (line 21) | async function upsertManyPosts(publicationID, newPosts, schemaField) {
function normalizePost (line 85) | function normalizePost(post) {
function normalizedDiff (line 95) | function normalizedDiff(existingPost, newPost) {
function postChanged (line 108) | function postChanged(existingPost, newPost) {
FILE: api/src/utils/urls.js
function getUrl (line 11) | function getUrl(urlName, ...args) {
function extractHostname (line 19) | function extractHostname(request) {
function ensureEncoded (line 45) | function ensureEncoded(url) {
FILE: api/src/utils/validation.js
function isURL (line 5) | function isURL(url) {
FILE: api/src/utils/watchdog.js
function startSampling (line 5) | function startSampling(metricName, sampleInterval = defaultSampleInterva...
FILE: api/src/workers/conductor.js
function forever (line 26) | function forever() {
function getPublications (line 47) | async function getPublications(
function conduct (line 71) | async function conduct() {
function shutdown (line 133) | function shutdown(signal) {
function failure (line 145) | function failure(reason, err) {
FILE: api/src/workers/og.js
function ogProcessor (line 27) | async function ogProcessor(job) {
function handleOg (line 76) | async function handleOg(job) {
function shutdown (line 203) | async function shutdown(signal) {
function failure (line 215) | async function failure(reason, err) {
FILE: api/src/workers/podcast.js
function sleep (line 41) | function sleep(time) {
function podcastProcessor (line 45) | async function podcastProcessor(job) {
function updateFeed (line 85) | async function updateFeed(podcastID, update, episodes) {
function handlePodcast (line 104) | async function handlePodcast(job) {
function markDone (line 275) | async function markDone(podcastID) {
function shutdown (line 283) | async function shutdown(signal) {
function failure (line 295) | async function failure(reason, err) {
FILE: api/src/workers/rss.js
function sleep (line 40) | function sleep(time) {
function rssProcessor (line 44) | async function rssProcessor(job) {
function updateFeed (line 86) | async function updateFeed(rssID, update, articles) {
function handleRSS (line 105) | async function handleRSS(job) {
function markDone (line 273) | async function markDone(rssID) {
function shutdown (line 278) | async function shutdown(signal) {
function failure (line 290) | async function failure(source, err) {
FILE: api/src/workers/social.js
function socialProcessor (line 46) | async function socialProcessor(job) {
function handleSocial (line 59) | async function handleSocial(job) {
function shutdown (line 114) | async function shutdown(signal) {
function failure (line 126) | async function failure(reason, err) {
FILE: api/src/workers/stream.js
function streamProcessor (line 23) | async function streamProcessor(job) {
function handleStream (line 55) | async function handleStream(job) {
function shutdown (line 79) | async function shutdown(signal) {
function failure (line 91) | async function failure(reason, err) {
FILE: api/src/workers/winds-hackernews.js
function tryHackernewsAPI (line 9) | async function tryHackernewsAPI(path, retries = 5) {
function hackernewsData (line 21) | async function hackernewsData(postID) {
function hackernewsTop (line 26) | async function hackernewsTop() {
function commentUrl (line 31) | function commentUrl(postID) {
function main (line 35) | async function main() {
FILE: api/src/workers/winds-queue-state-monitor.js
function countSetElements (line 9) | function countSetElements(key) {
function countLockTypes (line 13) | function countLockTypes(pattern) {
function main (line 29) | async function main() {
FILE: api/test-entry.js
function spyOnEverything (line 19) | function spyOnEverything(module) {
FILE: api/test/controllers/opml.js
function AuthGetRequest (line 28) | function AuthGetRequest(getPath) {
function AuthPostRequest (line 32) | function AuthPostRequest(path) {
FILE: api/test/utilities/upsert.js
function objectifyAndStripId (line 19) | function objectifyAndStripId(post) {
FILE: api/test/utils.js
function createMockFeed (line 21) | function createMockFeed(group, id) {
function getMockFeed (line 35) | function getMockFeed(group, id) {
function setupMocks (line 39) | function setupMocks() {
function getMockClient (line 48) | function getMockClient() {
function getTestFeed (line 56) | function getTestFeed(name) {
function getTestPodcast (line 60) | function getTestPodcast(name) {
function getTestPage (line 64) | function getTestPage(name) {
function loadFixture (line 68) | async function loadFixture(...fixtures) {
function withLogin (line 127) | function withLogin(
function dropDBs (line 138) | async function dropDBs() {
FILE: api/test/workers/conductor.js
function beforeDeadline (line 11) | function beforeDeadline() {
function afterDeadline (line 17) | function afterDeadline() {
FILE: api/test/workers/og.js
function setupHandler (line 15) | function setupHandler() {
FILE: api/test/workers/podcast.js
function setupHandler (line 22) | function setupHandler() {
FILE: api/test/workers/rss.js
function setupHandler (line 18) | function setupHandler() {
function queue (line 101) | async function queue(url) {
FILE: api/test/workers/social.js
function setupHandler (line 14) | function setupHandler() {
FILE: api/test/workers/stream.js
function setupHandler (line 13) | function setupHandler() {
FILE: app/public/actions.js
function init (line 4) | function init() {
FILE: app/public/electron.js
method click (line 102) | click() {
method click (line 108) | click() {
method click (line 156) | click() {
FILE: app/src/AppRouter.js
class AppRouter (line 19) | class AppRouter extends Component {
method componentDidMount (line 20) | componentDidMount() {
method render (line 24) | render() {
FILE: app/src/AuthedRoute.js
class AuthedRoute (line 7) | class AuthedRoute extends React.Component {
method render (line 8) | render() {
FILE: app/src/components/AddOPMLModal.js
class AddOPMLModal (line 14) | class AddOPMLModal extends React.Component {
method constructor (line 15) | constructor(props) {
method render (line 63) | render() {
FILE: app/src/components/AddPodcastModal.js
class AddPodcastModal (line 13) | class AddPodcastModal extends React.Component {
method constructor (line 14) | constructor(props) {
method render (line 104) | render() {
FILE: app/src/components/AddRSSModal.js
class AddRSSModal (line 14) | class AddRSSModal extends React.Component {
method constructor (line 15) | constructor(props) {
FILE: app/src/components/AliasModal.js
class AliasModal (line 12) | class AliasModal extends React.Component {
method constructor (line 13) | constructor(props) {
method render (line 48) | render() {
FILE: app/src/components/AllArticlesList.js
class AllArticles (line 10) | class AllArticles extends React.Component {
method constructor (line 11) | constructor(props) {
method componentDidMount (line 22) | componentDidMount() {
method componentDidUpdate (line 30) | componentDidUpdate() {
method componentWillUnmount (line 38) | componentWillUnmount() {
method getArticleFeed (line 42) | getArticleFeed() {
method render (line 46) | render() {
FILE: app/src/components/AllEpisodesList.js
class AllEpisodesList (line 11) | class AllEpisodesList extends React.Component {
method constructor (line 12) | constructor(props) {
method componentDidMount (line 24) | componentDidMount() {
method componentDidUpdate (line 32) | componentDidUpdate() {
method componentWillUnmount (line 40) | componentWillUnmount() {
method getEpisodeFeed (line 44) | getEpisodeFeed() {
method render (line 48) | render() {
FILE: app/src/components/ArticleListItem.js
class ArticleListItem (line 9) | class ArticleListItem extends React.Component {
method render (line 10) | render() {
FILE: app/src/components/BookmarkPanel.js
class BookmarkPanel (line 13) | class BookmarkPanel extends React.Component {
method componentDidMount (line 14) | componentDidMount() {
method render (line 19) | render() {
FILE: app/src/components/Drawer.js
class Drawer (line 4) | class Drawer extends React.Component {
method constructor (line 5) | constructor(props) {
method render (line 10) | render() {
FILE: app/src/components/EpisodeListItem.js
class EpisodeListItem (line 9) | class EpisodeListItem extends React.Component {
method render (line 16) | render() {
FILE: app/src/components/FeaturedItems.js
class FeaturedItems (line 8) | class FeaturedItems extends React.Component {
method constructor (line 9) | constructor(props) {
method componentDidMount (line 17) | componentDidMount() {
method render (line 21) | render() {
FILE: app/src/components/FeedHeader.js
class FeedHeader (line 15) | class FeedHeader extends React.Component {
method render (line 16) | render() {
FILE: app/src/components/FeedListItem.js
class FeedListItem (line 17) | class FeedListItem extends React.Component {
method render (line 18) | render() {
FILE: app/src/components/Folder/DeleteModal.js
class DeleteModal (line 7) | class DeleteModal extends React.Component {
method constructor (line 8) | constructor(props) {
method render (line 45) | render() {
FILE: app/src/components/Folder/FeedToFolderModal.js
class FeedToFolderModal (line 7) | class FeedToFolderModal extends React.Component {
method constructor (line 8) | constructor(props) {
method render (line 60) | render() {
FILE: app/src/components/Folder/Folder.js
class Folder (line 16) | class Folder extends React.Component {
method constructor (line 17) | constructor(props) {
method render (line 79) | render() {
FILE: app/src/components/Folder/FolderFeeds.js
class FolderFeeds (line 13) | class FolderFeeds extends React.Component {
method constructor (line 14) | constructor(props) {
method componentDidMount (line 29) | componentDidMount() {
method componentDidUpdate (line 41) | componentDidUpdate(prevProps) {
method subscribeToStreamFeed (line 56) | subscribeToStreamFeed(folderID, streamToken) {
method unsubscribeFromStreamFeed (line 63) | unsubscribeFromStreamFeed() {
method render (line 91) | render() {
FILE: app/src/components/Folder/FolderPanel.js
class FolderPanel (line 15) | class FolderPanel extends React.Component {
method constructor (line 16) | constructor(props) {
method render (line 28) | render() {
FILE: app/src/components/Folder/FolderPopover.js
class FolderPopover (line 14) | class FolderPopover extends React.Component {
method constructor (line 15) | constructor(props) {
method render (line 73) | render() {
FILE: app/src/components/Folder/IntroFolders.js
class IntroFolders (line 8) | class IntroFolders extends React.Component {
method constructor (line 9) | constructor(props) {
method render (line 23) | render() {
FILE: app/src/components/Folder/NewFolderModal.js
class NewFolderModal (line 13) | class NewFolderModal extends React.Component {
method constructor (line 14) | constructor(props) {
method render (line 76) | render() {
FILE: app/src/components/Folder/RenameModal.js
class RenameModal (line 10) | class RenameModal extends React.Component {
method constructor (line 11) | constructor(props) {
method render (line 44) | render() {
FILE: app/src/components/Folder/SearchFeed.js
class SearchFeed (line 15) | class SearchFeed extends React.Component {
method constructor (line 16) | constructor(props) {
method render (line 80) | render() {
FILE: app/src/components/Header.js
class Header (line 23) | class Header extends Component {
method constructor (line 24) | constructor(props) {
method render (line 106) | render() {
FILE: app/src/components/HtmlRender.js
class HtmlRender (line 21) | class HtmlRender extends React.Component {
method constructor (line 22) | constructor(props) {
method componentDidMount (line 40) | componentDidMount() {
method componentDidUpdate (line 52) | componentDidUpdate(prevProps) {
method componentWillUnmount (line 60) | componentWillUnmount() {
method render (line 240) | render() {
FILE: app/src/components/Notes/HighlightMenu.js
class HighlightMenu (line 4) | class HighlightMenu extends React.Component {
method constructor (line 5) | constructor(props) {
method render (line 21) | render() {
FILE: app/src/components/Notes/NoteInput.js
class NoteInput (line 6) | class NoteInput extends React.Component {
method render (line 28) | render() {
FILE: app/src/components/Notes/RecentNotesPanel.js
class RecentNotesPanel (line 13) | class RecentNotesPanel extends React.Component {
method render (line 14) | render() {
FILE: app/src/components/Panel.js
class Panel (line 5) | class Panel extends React.Component {
method constructor (line 6) | constructor(props) {
method render (line 12) | render() {
FILE: app/src/components/Player.js
class Player (line 20) | class Player extends Component {
method constructor (line 21) | constructor(props) {
method componentDidMount (line 37) | componentDidMount() {
method componentWillUnmount (line 42) | componentWillUnmount() {
method componentDidUpdate (line 50) | componentDidUpdate(prevProps) {
method render (line 239) | render() {
FILE: app/src/components/PodcastEpisode.js
class PodcastEpisode (line 14) | class PodcastEpisode extends React.Component {
method constructor (line 15) | constructor(props) {
method componentDidMount (line 29) | componentDidMount() {
method componentDidUpdate (line 33) | componentDidUpdate(prevProps) {
method render (line 125) | render() {
FILE: app/src/components/PodcastEpisodesView.js
class PodcastEpisodesView (line 21) | class PodcastEpisodesView extends React.Component {
method constructor (line 22) | constructor(props) {
method subscribeToStreamFeed (line 40) | subscribeToStreamFeed(podcastID, streamFeedToken) {
method unsubscribeFromStreamFeed (line 48) | unsubscribeFromStreamFeed() {
method componentDidMount (line 52) | componentDidMount() {
method componentDidUpdate (line 64) | componentDidUpdate(prevProps) {
method componentWillUnmount (line 80) | componentWillUnmount() {
method render (line 165) | render() {
FILE: app/src/components/PodcastPanels/PodcastList.js
class PodcastList (line 10) | class PodcastList extends React.Component {
method render (line 11) | render() {
FILE: app/src/components/PodcastPanels/RecentEpisodesPanel.js
class RecentEpisodesPanel (line 11) | class RecentEpisodesPanel extends React.Component {
method componentDidMount (line 12) | componentDidMount() {
method render (line 15) | render() {
FILE: app/src/components/PodcastPanels/SuggestedPodcasts.js
class SuggestedPodcasts (line 10) | class SuggestedPodcasts extends React.Component {
method componentDidMount (line 11) | componentDidMount() {
method render (line 16) | render() {
FILE: app/src/components/RSSArticle.js
function mergeSocialScore (line 15) | function mergeSocialScore(article, socialScore) {
class RSSArticle (line 26) | class RSSArticle extends React.Component {
method constructor (line 27) | constructor(props) {
method componentDidMount (line 43) | componentDidMount() {
method componentDidUpdate (line 56) | componentDidUpdate(prevProps) {
method render (line 165) | render() {
FILE: app/src/components/RSSArticleList.js
class RSSArticleList (line 21) | class RSSArticleList extends React.Component {
method constructor (line 22) | constructor(props) {
method subscribeToStreamFeed (line 42) | subscribeToStreamFeed(rssFeedID, streamFeedToken) {
method unsubscribeFromStreamFeed (line 50) | unsubscribeFromStreamFeed() {
method componentDidMount (line 54) | componentDidMount() {
method componentDidUpdate (line 67) | componentDidUpdate(prevProps) {
method componentWillUnmount (line 88) | componentWillUnmount() {
method render (line 177) | render() {
FILE: app/src/components/RSSPanels/RecentArticlesPanel.js
class RecentArticlesPanel (line 11) | class RecentArticlesPanel extends React.Component {
method componentDidMount (line 12) | componentDidMount() {
method render (line 16) | render() {
FILE: app/src/components/RSSPanels/RssFeedList.js
class RssFeedList (line 10) | class RssFeedList extends React.Component {
method render (line 11) | render() {
FILE: app/src/components/RSSPanels/SuggestedFeeds.js
class SuggestedFeeds (line 10) | class SuggestedFeeds extends React.Component {
method componentDidMount (line 11) | componentDidMount() {
method render (line 15) | render() {
FILE: app/src/components/SearchBar.js
class SearchBar (line 45) | class SearchBar extends React.Component {
method constructor (line 46) | constructor(props) {
method render (line 131) | render() {
FILE: app/src/components/SimpleProgressBar.js
class SimpleProgressBar (line 6) | class SimpleProgressBar extends React.Component {
method render (line 7) | render() {
FILE: app/src/components/Tabs.js
class Tabs (line 4) | class Tabs extends React.Component {
method constructor (line 5) | constructor(props) {
method render (line 12) | render() {
FILE: app/src/components/Tag/DeleteModal.js
class DeleteModal (line 7) | class DeleteModal extends React.Component {
method constructor (line 8) | constructor(props) {
method render (line 41) | render() {
FILE: app/src/components/Tag/RenameModal.js
class RenameModal (line 10) | class RenameModal extends React.Component {
method constructor (line 11) | constructor(props) {
method render (line 44) | render() {
FILE: app/src/components/Tag/Tag.js
class Tag (line 15) | class Tag extends React.Component {
method constructor (line 16) | constructor(props) {
method render (line 70) | render() {
FILE: app/src/components/Tag/TagFeeds.js
class TagFeeds (line 9) | class TagFeeds extends React.Component {
method render (line 10) | render() {
FILE: app/src/components/Tag/TagPanel.js
class TagPanel (line 12) | class TagPanel extends React.Component {
method render (line 13) | render() {
FILE: app/src/components/Tag/TagView.js
class TagView (line 16) | class TagView extends React.Component {
method constructor (line 17) | constructor(props) {
method render (line 78) | render() {
FILE: app/src/components/UserProfileSettingsDrawer.js
class UserProfileSettingsDrawer (line 15) | class UserProfileSettingsDrawer extends React.Component {
method constructor (line 16) | constructor(props) {
method componentDidMount (line 42) | componentDidMount() {
method componentDidUpdate (line 46) | componentDidUpdate(prevProps) {
method render (line 152) | render() {
FILE: app/src/reducers.js
function generateNotesOrder (line 284) | function generateNotesOrder(notes, sorted = false) {
FILE: app/src/serviceWorker.js
function register (line 23) | function register(config) {
function registerValidSW (line 57) | function registerValidSW(swUrl, config) {
function checkValidServiceWorker (line 101) | function checkValidServiceWorker(swUrl, config) {
function unregister (line 129) | function unregister() {
FILE: app/src/util/social.js
function extractRedditPostID (line 7) | function extractRedditPostID(article) {
function extractHackernewsPostID (line 27) | function extractHackernewsPostID(article) {
function refreshAccessToken (line 38) | async function refreshAccessToken() {
function sleep (line 60) | function sleep(time) {
function tryRedditAPI (line 64) | async function tryRedditAPI(path, retries = 3, backoffDelay = 20) {
function tryHackernewsAPI (line 95) | async function tryHackernewsAPI(path, retries = 3, backoffDelay = 20) {
function tryHackernewsSearch (line 111) | async function tryHackernewsSearch(query, retries = 3, backoffDelay = 20) {
function redditPost (line 133) | async function redditPost(article) {
function hackernewsPost (line 145) | async function hackernewsPost(article) {
function redditScore (line 157) | async function redditScore(postID) {
function hackernewsScore (line 165) | async function hackernewsScore(postID) {
function fetchSocialScore (line 182) | async function fetchSocialScore(source, article) {
FILE: app/src/views/AdminView.js
function compareTitles (line 8) | function compareTitles(lhs, rhs) {
class AdminView (line 12) | class AdminView extends React.Component {
method constructor (line 13) | constructor(props) {
method componentDidMount (line 24) | componentDidMount() {
method getRssFeeds (line 29) | getRssFeeds() {
method getPodcasts (line 35) | getPodcasts() {
method render (line 41) | render() {
class PodcastRow (line 187) | class PodcastRow extends React.Component {
method constructor (line 188) | constructor(props) {
method render (line 198) | render() {
class RssRow (line 328) | class RssRow extends React.Component {
method constructor (line 329) | constructor(props) {
method render (line 339) | render() {
FILE: app/src/views/Dashboard.js
class Dashboard (line 16) | class Dashboard extends React.Component {
method render (line 17) | render() {
FILE: app/src/views/FoldersView.js
class FoldersView (line 18) | class FoldersView extends React.Component {
method render (line 19) | render() {
FILE: app/src/views/PodcastsView.js
class PodcastsView (line 15) | class PodcastsView extends React.Component {
method constructor (line 16) | constructor(props) {
method componentDidMount (line 26) | componentDidMount() {
method componentDidUpdate (line 36) | componentDidUpdate(prevProps) {
method render (line 50) | render() {
FILE: app/src/views/RSSFeedsView.js
class RSSFeedsView (line 15) | class RSSFeedsView extends React.Component {
method constructor (line 16) | constructor(props) {
method componentDidMount (line 26) | componentDidMount() {
method componentDidUpdate (line 36) | componentDidUpdate(prevProps) {
method render (line 50) | render() {
FILE: app/src/views/auth-views/Create.js
class Create (line 12) | class Create extends Component {
method constructor (line 13) | constructor(props) {
method render (line 21) | render() {
class OnboardingGrid (line 59) | class OnboardingGrid extends React.Component {
method constructor (line 60) | constructor(props) {
method toggleInterest (line 68) | toggleInterest(interestName) {
method render (line 86) | render() {
class AccountDetailsForm (line 154) | class AccountDetailsForm extends React.Component {
method constructor (line 155) | constructor(props) {
method render (line 239) | render() {
FILE: app/src/views/auth-views/ForgotPassword.js
class ForgotPassword (line 9) | class ForgotPassword extends Component {
method constructor (line 10) | constructor(props) {
method validateEmail (line 23) | validateEmail(e) {
method requestResetPasscode (line 42) | requestResetPasscode(e) {
method render (line 61) | render() {
FILE: app/src/views/auth-views/Login.js
class Login (line 9) | class Login extends Component {
method constructor (line 10) | constructor(props) {
method render (line 52) | render() {
FILE: app/src/views/auth-views/ResetPassword.js
class ResetPassword (line 9) | class ResetPassword extends Component {
method constructor (line 10) | constructor(props) {
method validateForm (line 28) | validateForm() {
method validateEmail (line 36) | validateEmail(e) {
method validateRecoveryCode (line 58) | validateRecoveryCode(e) {
method validatePassword (line 82) | validatePassword(e) {
method requestPasswordReset (line 106) | requestPasswordReset(e) {
method render (line 125) | render() {
Condensed preview — 445 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (8,200K chars).
[
{
"path": ".eslintrc.js",
"chars": 824,
"preview": "module.exports = {\n\tenv: {\n\t\t'browser': true,\n\t\t'es6': true,\n\t\t'node': true,\n\t\t'shared-node-browser': true,\n\t\t'mocha': t"
},
{
"path": ".gitattributes",
"chars": 49,
"preview": "* linguist-vendored\n*.js linguist-vendored=false\n"
},
{
"path": ".github/ISSUE_TEMPLATE.md",
"chars": 1042,
"preview": "<!--\n\nHave you read Winds Code of Conduct? By filing an Issue, you are expected to comply with it, including treating ev"
},
{
"path": ".github/PULL_REQUEST_TEMPLATE.md",
"chars": 1217,
"preview": "### Requirements\r\n\r\n* Filling out the template is required. Any pull request that does not include enough information to"
},
{
"path": ".github/stale.yml",
"chars": 683,
"preview": "# Number of days of inactivity before an issue becomes stale\ndaysUntilStale: 30\n# Number of days of inactivity before a "
},
{
"path": ".gitignore",
"chars": 1409,
"preview": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directo"
},
{
"path": ".prettierrc",
"chars": 221,
"preview": "{\n \"useTabs\": true,\n \"printWidth\": 90,\n \"tabWidth\": 4,\n \"singleQuote\": true,\n \"trailingComma\": \"all\",\n "
},
{
"path": ".travis.yml",
"chars": 204,
"preview": "dist: xenial\nlanguage: node_js\nnode_js:\n - \"14\"\nservices:\n - redis\n - mongodb\ncache: yarn\ninstall:\n - \"cd api && y"
},
{
"path": "Dockerfile",
"chars": 424,
"preview": "# Use the latest version of Node\nFROM mhart/alpine-node:latest\n\n# Update dependency cache\nRUN apk update && apk upgrade\n"
},
{
"path": "LICENSE",
"chars": 1471,
"preview": "Copyright 2018 Stream.io Inc\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without mo"
},
{
"path": "README.md",
"chars": 19042,
"preview": "> 🛑 **Notice**: This repository is no longer maintained; No further Issues or Pull Requests will be considered or approv"
},
{
"path": "RSS.md",
"chars": 1880,
"preview": "\n## Post Uniqueness ##\n\nPost uniqueness in an RSS feed can be determined by 4 different methods:\n\n- The guid property on"
},
{
"path": "STYLE.md",
"chars": 242,
"preview": "The style rules for Winds are defined in:\n\n* .prettierrc\n* .eslintrc\n\nTo cleanup your code run\n\n`yarn prettier` in t"
},
{
"path": "api/Dockerfile",
"chars": 474,
"preview": "# Use the latest version of Node\nFROM mhart/alpine-node:latest\n\n# Update dependency cache\nRUN apk update && apk upgrade\n"
},
{
"path": "api/build.sh",
"chars": 490,
"preview": "#!/bin/bash\n\ncd api\n\n# Remove the existing build directory and create a fresh one\nrm -rf dist && mkdir -p dist/{utils,em"
},
{
"path": "api/config.yaml",
"chars": 235,
"preview": "\napiVersion: v1\nkind: Service\nmetadata:\n name: api\n annotations:\n cloud.google.com/load-balancer-type: \"Internal\"\n "
},
{
"path": "api/docker-compose.yml",
"chars": 1423,
"preview": "version: '3.7'\nservices:\n api:\n build:\n context: .\n dockerfile: Dockerfile\n volumes:\n - '/opt/data"
},
{
"path": "api/ecosystem.dev.config.js",
"chars": 936,
"preview": "module.exports = {\n\tapps: [\n\t\t{\n\t\t\tname: 'api',\n\t\t\tinterpreter: 'babel-node',\n\t\t\tscript: 'src/server.js',\n\t\t\twatch: true"
},
{
"path": "api/now.json",
"chars": 62,
"preview": "{\n \"type\": \"docker\",\n \"features\": {\n \"cloud\": \"v2\"\n }\n}\n"
},
{
"path": "api/package.json",
"chars": 3169,
"preview": "{\n\t\"name\": \"api\",\n\t\"description\": \"https://getstream.io/winds\",\n\t\"license\": \"BSD-3-Clause\",\n\t\"scripts\": {\n\t\t\"preinstall\""
},
{
"path": "api/scripts/docker-build.sh",
"chars": 167,
"preview": "#!/bin/bash\n\n# Build the Docker image\ndocker build -t winds-api .\n\n# List all Docker images\ndocker images\n\n# Run the Doc"
},
{
"path": "api/scripts/docker-compose-aws.sh",
"chars": 88,
"preview": "#!/bin/bash\n\n# Build with Docker Compose\ndocker-compose -f ../docker-compose-aws.yml up\n"
},
{
"path": "api/scripts/docker-compose.sh",
"chars": 84,
"preview": "#!/bin/bash\n\n# Build with Docker Compose\ndocker-compose -f ../docker-compose.yml up\n"
},
{
"path": "api/scripts/make-build.sh",
"chars": 578,
"preview": "#!/bin/bash\n\n# Remove the existing build directory and create a fresh one\nrm -rf ../dist && mkdir ../dist\n\n# Transpile E"
},
{
"path": "api/setup-tests.js",
"chars": 2587,
"preview": "import fs from 'fs';\nimport path from 'path';\nimport Mocha from 'mocha';\nimport chai from 'chai';\nimport chaiHttp from '"
},
{
"path": "api/src/.babelrc",
"chars": 104,
"preview": "{\n\t\"plugins\": [ \"shebang\"],\n\t\"presets\": [[\"@babel/preset-env\", { \"targets\": { \"node\": \"current\" } }]]\n}\n"
},
{
"path": "api/src/.prettierignore",
"chars": 16,
"preview": "config/index.js\n"
},
{
"path": "api/src/asyncTasks.js",
"chars": 4403,
"preview": "import config from './config';\nimport logger from './utils/logger';\n\nimport Queue from 'bull';\nimport { getStatsDClient "
},
{
"path": "api/src/commands/_debug-feed.js",
"chars": 4879,
"preview": "import program from 'commander';\nimport '../loadenv';\nimport '../utils/db';\nimport { ParseFeed, ParsePodcast } from '../"
},
{
"path": "api/src/commands/cleanup-follows.js",
"chars": 1770,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/denormalize-follows.js",
"chars": 2091,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/denormalize-pins.js",
"chars": 2269,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/email.js",
"chars": 1638,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/load-featured-feeds.js",
"chars": 7183,
"preview": "import '../loadenv';\n\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport fs from 'fs';\nimport"
},
{
"path": "api/src/commands/rescrape-favicon.js",
"chars": 2896,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/rescrape-og.js",
"chars": 2810,
"preview": "import '../utils/db';\nimport program from 'commander';\nimport Podcast from '../models/podcast';\nimport Article from '../"
},
{
"path": "api/src/commands/reset-parsing-state.js",
"chars": 942,
"preview": "import '../loadenv';\n\nimport RSS from '../models/rss';\nimport Podcast from '../models/podcast';\n\nimport '../utils/db';\ni"
},
{
"path": "api/src/commands/resync-follows.js",
"chars": 1590,
"preview": "import '../loadenv';\nimport '../utils/db';\nimport program from 'commander';\nimport logger from '../utils/logger';\nimport"
},
{
"path": "api/src/commands/winds-article.js",
"chars": 748,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport program from 'commander';\nimport chalk from 'chalk';\nimport logger fr"
},
{
"path": "api/src/commands/winds-discover.js",
"chars": 1199,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport program from 'commander';\nimport chalk from 'chalk';\nimport logger fr"
},
{
"path": "api/src/commands/winds-merge.js",
"chars": 4775,
"preview": "import mongoose from 'mongoose';\nimport ProgressBar from 'progress';\n\nimport db from '../utils/db';\nimport { upsertColle"
},
{
"path": "api/src/commands/winds-og.js",
"chars": 1713,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport program from 'commander';\nimport chalk from 'chalk';\nimport logger fr"
},
{
"path": "api/src/commands/winds-podcast.js",
"chars": 325,
"preview": "import program from 'commander';\nimport '../loadenv';\nimport '../utils/db';\nimport { debugFeed } from './_debug-feed';\n\n"
},
{
"path": "api/src/commands/winds-rebuild-search.js",
"chars": 1429,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport logger from '../utils/logger';\nimport Podcast from '../models/podcast"
},
{
"path": "api/src/commands/winds-rehash.js",
"chars": 912,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport logger from '../utils/logger';\nimport Article from '../models/article"
},
{
"path": "api/src/commands/winds-rss.js",
"chars": 321,
"preview": "import program from 'commander';\nimport '../loadenv';\nimport '../utils/db';\nimport { debugFeed } from './_debug-feed';\n\n"
},
{
"path": "api/src/commands/winds-truncate-rss-feed.js",
"chars": 511,
"preview": "import '../loadenv';\nimport '../utils/db';\n\nimport RSS from '../models/rss';\nimport Article from '../models/article';\nim"
},
{
"path": "api/src/commands/winds.js",
"chars": 724,
"preview": "#!/usr/bin/env babel-node\nimport program from 'commander';\nimport logger from '../utils/logger';\n\nlet version;\n\nif (proc"
},
{
"path": "api/src/config/dev.js",
"chars": 222,
"preview": "module.exports = {\n\turl: 'https://winds.getstream.io',\n\tlogger: { level: process.env.LOGGER_LEVEL || 'info' },\n\temail: {"
},
{
"path": "api/src/config/index.js",
"chars": 2340,
"preview": "import dotenv from 'dotenv';\nimport path from 'path';\n\nconst configs = {\n\tdevelopment: { config: 'dev' },\n\tproduction: {"
},
{
"path": "api/src/config/prod.js",
"chars": 21,
"preview": "module.exports = {};\n"
},
{
"path": "api/src/config/test.js",
"chars": 547,
"preview": "module.exports = {\n\tdatabase: {\n\t\turi: 'mongodb://localhost:27017/test',\n\t},\n\tcache: {\n\t\turi: 'redis://localhost:6379/10"
},
{
"path": "api/src/controllers/alias.js",
"chars": 2810,
"preview": "import mongoose from 'mongoose';\n\nimport Alias from '../models/alias';\nimport Rss from '../models/rss';\nimport Podcast f"
},
{
"path": "api/src/controllers/article.js",
"chars": 1519,
"preview": "import mongoose from 'mongoose';\n\nimport Article from '../models/article';\nimport { getArticleRecommendations } from '.."
},
{
"path": "api/src/controllers/auth.js",
"chars": 4319,
"preview": "import { v4 as uuidv4 } from 'uuid';\nimport validator from 'validator';\n\nimport User from '../models/user';\nimport RSS f"
},
{
"path": "api/src/controllers/default.js",
"chars": 130,
"preview": "exports.get = (req, res) => {\n\tres.status(200).send('pong');\n};\n\nexports.post = (req, res) => {\n\tres.status(200).send('p"
},
{
"path": "api/src/controllers/email.js",
"chars": 1816,
"preview": "import mongoose from 'mongoose';\n\nimport User from '../models/user';\n\nimport {\n\tdailyContextGlobal,\n\tdailyContextUser,\n\t"
},
{
"path": "api/src/controllers/episode.js",
"chars": 1637,
"preview": "import Episode from '../models/episode';\n\nimport { getEpisodeRecommendations } from '../utils/personalization';\nimport {"
},
{
"path": "api/src/controllers/featured.js",
"chars": 1037,
"preview": "import RSS from '../models/rss';\nimport Podcast from '../models/podcast';\n\nimport config from '../config';\n\nlet packageI"
},
{
"path": "api/src/controllers/feed.js",
"chars": 1544,
"preview": "import Article from '../models/article';\nimport Episode from '../models/episode';\n\nimport config from '../config';\nimpor"
},
{
"path": "api/src/controllers/folder.js",
"chars": 6545,
"preview": "import mongoose from 'mongoose';\nimport config from '../config';\n\nimport { getStreamClient } from '../utils/stream';\nimp"
},
{
"path": "api/src/controllers/follow.js",
"chars": 2525,
"preview": "import stream from 'getstream';\nimport async from 'async';\nimport mongoose from 'mongoose';\n\nimport Follow from '../mode"
},
{
"path": "api/src/controllers/health.js",
"chars": 4928,
"preview": "import Article from '../models/article';\nimport Episode from '../models/episode';\nimport RSS from '../models/rss';\nimpor"
},
{
"path": "api/src/controllers/listen.js",
"chars": 987,
"preview": "import Listen from '../models/listen';\nimport User from '../models/user';\nimport { trackEngagement } from '../utils/anal"
},
{
"path": "api/src/controllers/note.js",
"chars": 2432,
"preview": "import Note from '../models/note';\n\nexports.list = async (req, res) => {\n\tres.json(await Note.find({ user: req.user.sub "
},
{
"path": "api/src/controllers/opml.js",
"chars": 6404,
"preview": "import opmlParser from 'node-opml-parser';\nimport opmlGenerator from 'opml-generator';\nimport moment from 'moment';\nimpo"
},
{
"path": "api/src/controllers/pin.js",
"chars": 2228,
"preview": "import mongoose from 'mongoose';\n\nimport Pin from '../models/pin';\nimport config from '../config';\nimport { trackEngagem"
},
{
"path": "api/src/controllers/playlist.js",
"chars": 1841,
"preview": "import Playlist from '../models/playlist';\n\nimport config from '../config';\nimport search from '../utils/search';\n\nasync"
},
{
"path": "api/src/controllers/podcast.js",
"chars": 4618,
"preview": "import mongoose from 'mongoose';\nimport normalizeUrl from 'normalize-url';\n\nimport Podcast from '../models/podcast';\n\nim"
},
{
"path": "api/src/controllers/rss.js",
"chars": 4196,
"preview": "import mongoose from 'mongoose';\nimport moment from 'moment';\nimport normalizeUrl from 'normalize-url';\nimport entities "
},
{
"path": "api/src/controllers/tag.js",
"chars": 1938,
"preview": "import mongoose from 'mongoose';\n\nimport Tag from '../models/tag';\n\nexports.list = async (req, res) => {\n\tres.json(await"
},
{
"path": "api/src/controllers/user.js",
"chars": 2787,
"preview": "import validator from 'validator';\n\nimport User from '../models/user';\nimport RSS from '../models/rss';\nimport Podcast f"
},
{
"path": "api/src/fixtures/featured.json",
"chars": 20424,
"preview": "{\n\t\"podcasts\": [\n\t\t{\n\t\t\t\"category\": \"UI/UX\",\n\t\t\t\"name\": \"Design Details\",\n\t\t\t\"feedUrl\": \"https://rss.simplecast.com/podc"
},
{
"path": "api/src/loadenv.js",
"chars": 271,
"preview": "import dotenv from 'dotenv';\nimport path from 'path';\n\n// workaround based on https://github.com/motdotla/dotenv/issues/"
},
{
"path": "api/src/models/alias.js",
"chars": 985,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/article.js",
"chars": 4278,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/content.js",
"chars": 905,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/enclosure.js",
"chars": 413,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/episode.js",
"chars": 3981,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/folder.js",
"chars": 1362,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/follow.js",
"chars": 4252,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/listen.js",
"chars": 1149,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/note.js",
"chars": 1187,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/pin.js",
"chars": 1269,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/playlist.js",
"chars": 1018,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/podcast.js",
"chars": 4447,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/rss.js",
"chars": 4480,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/tag.js",
"chars": 981,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport timestamps from 'mongoose-timestamp';\nimport mongooseStringQuery fro"
},
{
"path": "api/src/models/user.js",
"chars": 3827,
"preview": "import mongoose, { Schema } from 'mongoose';\nimport bcrypt from 'mongoose-bcrypt';\nimport timestamps from 'mongoose-time"
},
{
"path": "api/src/parsers/content.js",
"chars": 128,
"preview": "import Mercury from '@postlight/mercury-parser';\n\nexport async function ParseContent(url) {\n\treturn await Mercury.parse("
},
{
"path": "api/src/parsers/detect-language.js",
"chars": 1205,
"preview": "import { ReadFeedStream, ReadFeedURL } from '../parsers/feed';\nimport franc from 'franc-min';\n\n// DetectLanguage returns"
},
{
"path": "api/src/parsers/detect-type.js",
"chars": 655,
"preview": "import { ReadFeedURL, ReadFeedStream } from './feed.js';\n\n// determines if the given feedStream is a podcast or not\nexpo"
},
{
"path": "api/src/parsers/discovery.js",
"chars": 6198,
"preview": "import request from 'request';\nimport normalize from 'normalize-url';\nimport url from 'url';\nimport FeedParser from 'fee"
},
{
"path": "api/src/parsers/feed.js",
"chars": 13734,
"preview": "import request from 'request';\nimport entities from 'entities';\nimport moment from 'moment';\nimport normalize from 'norm"
},
{
"path": "api/src/parsers/og.js",
"chars": 2220,
"preview": "import logger from '../utils/logger';\nimport { ReadPageURL } from './feed.js';\n\nconst invalidExtensions = ['mp3', 'mp4',"
},
{
"path": "api/src/routes/alias.js",
"chars": 410,
"preview": "import Alias from '../controllers/alias';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {"
},
{
"path": "api/src/routes/article.js",
"chars": 246,
"preview": "import Article from '../controllers/article';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) "
},
{
"path": "api/src/routes/auth.js",
"chars": 380,
"preview": "import Auth from '../controllers/auth';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {\n\t"
},
{
"path": "api/src/routes/email.js",
"chars": 293,
"preview": "import Email from '../controllers/email';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {"
},
{
"path": "api/src/routes/episode.js",
"chars": 246,
"preview": "import Episode from '../controllers/episode';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) "
},
{
"path": "api/src/routes/featured.js",
"chars": 185,
"preview": "import Featured from '../controllers/featured';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api"
},
{
"path": "api/src/routes/folder.js",
"chars": 487,
"preview": "import Folder from '../controllers/folder';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) =>"
},
{
"path": "api/src/routes/follow.js",
"chars": 288,
"preview": "import Follow from '../controllers/follow';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) =>"
},
{
"path": "api/src/routes/health.js",
"chars": 499,
"preview": "import Health from '../controllers/health';\nimport { wrapAsync } from '../utils/controllers';\nimport basicAuth from '../"
},
{
"path": "api/src/routes/index.js",
"chars": 148,
"preview": "import Default from '../controllers/default';\n\nmodule.exports = (api) => {\n\tapi.route('/').get(Default.get);\n\tapi.route("
},
{
"path": "api/src/routes/listen.js",
"chars": 231,
"preview": "import Listen from '../controllers/listen';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) =>"
},
{
"path": "api/src/routes/note.js",
"chars": 390,
"preview": "import Note from '../controllers/note';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {\n\t"
},
{
"path": "api/src/routes/opml.js",
"chars": 311,
"preview": "import multer from 'multer';\n\nimport OPML from '../controllers/opml';\nimport { wrapAsync } from '../utils/controllers';\n"
},
{
"path": "api/src/routes/pin.js",
"chars": 323,
"preview": "import Pin from '../controllers/pin';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {\n\tap"
},
{
"path": "api/src/routes/playlist.js",
"chars": 450,
"preview": "import Playlist from '../controllers/playlist';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api"
},
{
"path": "api/src/routes/podcast.js",
"chars": 570,
"preview": "import Podcast from '../controllers/podcast';\nimport Episode from '../controllers/episode';\nimport { wrapAsync } from '."
},
{
"path": "api/src/routes/rss.js",
"chars": 500,
"preview": "import RSS from '../controllers/rss';\nimport Article from '../controllers/article';\nimport { wrapAsync } from '../utils/"
},
{
"path": "api/src/routes/tag.js",
"chars": 375,
"preview": "import Tag from '../controllers/tag';\nimport { wrapAsync } from '../utils/controllers';\n\nmodule.exports = (api) => {\n\tap"
},
{
"path": "api/src/routes/user.js",
"chars": 442,
"preview": "import User from '../controllers/user';\nimport Feed from '../controllers/feed';\nimport { wrapAsync } from '../utils/cont"
},
{
"path": "api/src/server.js",
"chars": 2283,
"preview": "import config from './config';\nimport fs from 'fs';\nimport path from 'path';\n\nimport express from 'express';\nimport body"
},
{
"path": "api/src/utils/analytics.js",
"chars": 943,
"preview": "import jwt from 'jsonwebtoken';\nimport streamAnalytics from 'stream-analytics';\n\nimport logger from './logger';\nimport c"
},
{
"path": "api/src/utils/basicAuth.js",
"chars": 510,
"preview": "import basicAuth from 'express-basic-auth';\nimport User from '../models/user';\n\nconst asyncAuthorizer = async (email, pa"
},
{
"path": "api/src/utils/blockedURLs.js",
"chars": 134,
"preview": "const blockedURLs = ['indeed.'];\n\nexport const isBlockedURLs = (url = '') => {\n\treturn !!blockedURLs.find((u) => url.inc"
},
{
"path": "api/src/utils/collections.js",
"chars": 2625,
"preview": "import Podcast from '../models/podcast';\nimport RSS from '../models/rss';\nimport Article from '../models/article';\nimpor"
},
{
"path": "api/src/utils/controllers.js",
"chars": 305,
"preview": "function wrapAsync(fn) {\n\treturn function wrapAsyncInner(req, res, next) {\n\t\t// Make sure to `.catch()` any errors and p"
},
{
"path": "api/src/utils/db/index.js",
"chars": 811,
"preview": "import mongoose from 'mongoose';\n\nimport config from '../../config';\nimport logger from '../logger';\n\nmongoose.Promise ="
},
{
"path": "api/src/utils/email/context.js",
"chars": 2550,
"preview": "import * as personalization from '../personalization';\nimport Podcast from '../../models/podcast';\nimport Article from '"
},
{
"path": "api/src/utils/email/send.js",
"chars": 2061,
"preview": "import fs from 'fs';\nimport ejs from 'ejs';\nimport sendgrid from '@sendgrid/mail';\n\nimport logger from '../logger';\nimpo"
},
{
"path": "api/src/utils/email/templates/daily.ejs",
"chars": 1775,
"preview": "<html>\n <head>\n <title>Winds Digest</title>\n </head>\n <body>\n Winds<br/>\n <%= user.usernam"
},
{
"path": "api/src/utils/email/templates/followee.ejs",
"chars": 401,
"preview": "<html>\n <head>\n <title>You're so popular! You have a new follower!</title>\n </head>\n <body>\n You'"
},
{
"path": "api/src/utils/email/templates/password.ejs",
"chars": 398,
"preview": "<html>\n <head>\n <title>Forgot Password</title>\n </head>\n <body>\n Looks like you really blew it, a"
},
{
"path": "api/src/utils/email/templates/weekly.ejs",
"chars": 1775,
"preview": "<html>\n <head>\n <title>Winds Digest</title>\n </head>\n <body>\n Winds<br/>\n <%= user.usernam"
},
{
"path": "api/src/utils/email/templates/welcome.ejs",
"chars": 916,
"preview": "<html>\n <head>\n <title>Welcome to Winds!</title>\n </head>\n <body>\n Thanks for checking out Winds."
},
{
"path": "api/src/utils/errors.js",
"chars": 1807,
"preview": "import config from '../config';\nimport Raven from 'raven';\nimport path from 'path';\nimport logger from '../utils/logger'"
},
{
"path": "api/src/utils/logger/index.js",
"chars": 1310,
"preview": "import winston from 'winston';\nimport { inspect } from 'util';\n\nimport config from '../../config';\nimport { createSentry"
},
{
"path": "api/src/utils/logger/sentry.js",
"chars": 2132,
"preview": "import Transport from 'winston-transport';\n\nconst winstonLevelToSentryLevel = {\n\tsilly: 'debug',\n\tverbose: 'debug',\n\tinf"
},
{
"path": "api/src/utils/merge.js",
"chars": 3906,
"preview": "import mongoose from 'mongoose';\n\nimport Article from '../models/article';\nimport Episode from '../models/episode';\nimpo"
},
{
"path": "api/src/utils/personalization/index.js",
"chars": 2986,
"preview": "import jwt from 'jsonwebtoken';\nimport Article from '../../models/article';\nimport Podcast from '../../models/podcast';\n"
},
{
"path": "api/src/utils/queue.js",
"chars": 2055,
"preview": "import Redis from 'ioredis';\n\nimport config from '../config';\n\nconst redis = new Redis(config.cache.uri);\n\nconst TTL = 3"
},
{
"path": "api/src/utils/random.js",
"chars": 381,
"preview": "// returns a random number from 2**1, 2**2, ..., 2**n-1, 2**n\n// 2 is two time more likely to be returned than 4, 4 than"
},
{
"path": "api/src/utils/rate-limiter.js",
"chars": 1765,
"preview": "import Redis from 'ioredis';\n\nimport config from '../config';\n\nconst redis = new Redis(config.cache.uri);\n\nredis.defineC"
},
{
"path": "api/src/utils/sanitize.js",
"chars": 652,
"preview": "import sanitizeHtml from 'sanitize-html';\n\nconst allowedTags = [\n\t'h2',\n\t'h3',\n\t'h4',\n\t'h5',\n\t'h6',\n\t'p',\n\t'a',\n\t'ul',\n\t"
},
{
"path": "api/src/utils/search/index.js",
"chars": 812,
"preview": "import algolia from 'algoliasearch';\nimport config from '../../config';\nimport util from 'util';\nimport logger from '../"
},
{
"path": "api/src/utils/social.js",
"chars": 5147,
"preview": "import request from 'request-promise-native';\nimport * as urlParser from 'url';\nimport querystring from 'querystring';\n\n"
},
{
"path": "api/src/utils/statsd.js",
"chars": 532,
"preview": "import config from '../config';\n\nimport { StatsD } from 'node-statsd';\n\nvar statsDClient = null;\n\nfunction getStatsDClie"
},
{
"path": "api/src/utils/stream.js",
"chars": 264,
"preview": "import stream from 'getstream';\nimport config from '../config';\n\nvar streamClient = null;\n\nexport function getStreamClie"
},
{
"path": "api/src/utils/upsert.js",
"chars": 3510,
"preview": "import { diff } from 'deep-object-diff';\n\nimport Article from '../models/article';\nimport Episode from '../models/episod"
},
{
"path": "api/src/utils/urls.js",
"chars": 1218,
"preview": "import config from '../config';\nimport util from 'util';\n\nconst urlMap = {\n\tarticle_detail: 'rss/%s/articles/%s',\n\tepiso"
},
{
"path": "api/src/utils/validation.js",
"chars": 727,
"preview": "import normalizeUrl from 'normalize-url';\nimport validator from 'validator';\nimport logger from './logger';\n\nexport func"
},
{
"path": "api/src/utils/watchdog.js",
"chars": 657,
"preview": "import { getStatsDClient } from './statsd';\n\nconst defaultSampleInterval = 60;\n\nexport function startSampling(metricName"
},
{
"path": "api/src/workers/conductor.js",
"chars": 4533,
"preview": "import moment from 'moment';\nimport mongoose from 'mongoose';\n\nimport db from '../utils/db';\n\nimport RSS from '../models"
},
{
"path": "api/src/workers/og.js",
"chars": 6444,
"preview": "import joi from 'joi';\nimport mongoose from 'mongoose';\nimport normalize from 'normalize-url';\nimport { EventEmitter } f"
},
{
"path": "api/src/workers/podcast.js",
"chars": 8297,
"preview": "import joi from 'joi';\nimport moment from 'moment';\nimport mongoose from 'mongoose';\nimport { EventEmitter } from 'event"
},
{
"path": "api/src/workers/rss.js",
"chars": 8301,
"preview": "import joi from 'joi';\nimport moment from 'moment';\nimport mongoose from 'mongoose';\nimport { EventEmitter } from 'event"
},
{
"path": "api/src/workers/social.js",
"chars": 3959,
"preview": "import mongoose from 'mongoose';\nimport joi from 'joi';\n\nimport db from '../utils/db';\n\nimport RSS from '../models/rss';"
},
{
"path": "api/src/workers/stream.js",
"chars": 3022,
"preview": "import joi from 'joi';\nimport mongoose from 'mongoose';\n\nimport db from '../utils/db';\n\nimport RSS from '../models/rss';"
},
{
"path": "api/src/workers/winds-hackernews.js",
"chars": 1927,
"preview": "import request from 'request-promise-native';\n\nimport '../config';\nimport db from '../utils/db';\n\nimport Rss from '../mo"
},
{
"path": "api/src/workers/winds-queue-state-monitor.js",
"chars": 1760,
"preview": "import Redis from 'ioredis';\n\nimport config from '../config';\nimport { getStatsDClient } from '../utils/statsd';\n\nconst "
},
{
"path": "api/test/controllers/alias.js",
"chars": 2495,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Alias from '../../src/models/alias';"
},
{
"path": "api/test/controllers/article.js",
"chars": 1950,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Article from '../../src/models/artic"
},
{
"path": "api/test/controllers/auth.js",
"chars": 12338,
"preview": "import { expect, request } from 'chai';\nimport jwt from 'jsonwebtoken';\n\nimport api from '../../src/server';\nimport conf"
},
{
"path": "api/test/controllers/episode.js",
"chars": 2143,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Episode from '../../src/models/episo"
},
{
"path": "api/test/controllers/featured.js",
"chars": 544,
"preview": "import { expect, request } from 'chai';\n\nimport { withLogin } from '../utils.js';\nimport api from '../../src/server';\nim"
},
{
"path": "api/test/controllers/feed.js",
"chars": 2748,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport User from '../../src/models/user';\ni"
},
{
"path": "api/test/controllers/folder.js",
"chars": 7082,
"preview": "import { expect, request } from 'chai';\nimport { dropDBs, loadFixture, withLogin } from '../utils';\n\nimport Folder from "
},
{
"path": "api/test/controllers/follow.js",
"chars": 6024,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Follow from '../../src/models/follow"
},
{
"path": "api/test/controllers/health.js",
"chars": 747,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\n\nimport { loadFixture, withLogin, dropDBs }"
},
{
"path": "api/test/controllers/listen.js",
"chars": 2976,
"preview": "import sinon from 'sinon';\nimport { expect, request } from 'chai';\n\nimport User from '../../src/models/user';\nimport Lis"
},
{
"path": "api/test/controllers/note.js",
"chars": 7350,
"preview": "import { expect, request } from 'chai';\nimport { dropDBs, loadFixture, withLogin } from '../utils';\n\nimport Note from '."
},
{
"path": "api/test/controllers/opml.js",
"chars": 4689,
"preview": "import { expect, request } from 'chai';\nimport fs from 'fs';\nimport path from 'path';\n\nimport api from '../../src/server"
},
{
"path": "api/test/controllers/pin.js",
"chars": 2131,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Pin from '../../src/models/pin';\nimp"
},
{
"path": "api/test/controllers/playlist.js",
"chars": 8681,
"preview": "import sinon from 'sinon';\nimport { expect, request } from 'chai';\n\nimport User from '../../src/models/user';\nimport Pla"
},
{
"path": "api/test/controllers/podcast.js",
"chars": 2803,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport Podcast from '../../src/models/podca"
},
{
"path": "api/test/controllers/rss.js",
"chars": 3621,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport RSS from '../../src/models/rss';\nimp"
},
{
"path": "api/test/controllers/tag.js",
"chars": 6427,
"preview": "import { expect, request } from 'chai';\nimport { dropDBs, loadFixture, withLogin } from '../utils';\n\nimport Tag from '.."
},
{
"path": "api/test/controllers/user.js",
"chars": 7192,
"preview": "import { expect, request } from 'chai';\n\nimport api from '../../src/server';\nimport User from '../../src/models/user';\ni"
},
{
"path": "api/test/data/404.opml",
"chars": 560,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<opml version=\"1.0\">\n <head>\n <title>Subscriptions</title>\n </head>\n <body>"
},
{
"path": "api/test/data/discovery/case.html",
"chars": 249,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <link rel=\"shortcut icon\" href=\"favicon.ico\">\n <link rel=\"alternate\" type=\"applicat"
},
{
"path": "api/test/data/discovery/fail.xml",
"chars": 199,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <meta charset=\"UTF-8\">\n <feed></feed>\n <title>The Is Not The Feed You're Looking"
},
{
"path": "api/test/data/discovery/index.html",
"chars": 249,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <link rel=\"shortcut icon\" href=\"favicon.ico\">\n <link rel=\"alternate\" type=\"applicat"
},
{
"path": "api/test/data/discovery/nofavicon.html",
"chars": 199,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS\" href=\"/rssfinder.xml\">\n "
},
{
"path": "api/test/data/discovery/nourl.xml",
"chars": 1505,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n "
},
{
"path": "api/test/data/discovery/rss.xml",
"chars": 110219,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet type=\"text/xsl\" media=\"screen\" href=\"/~d/styles/rss2full.xsl\"?>"
},
{
"path": "api/test/data/feed/90.cx",
"chars": 26747,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
},
{
"path": "api/test/data/feed/a16z",
"chars": 12566,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
},
{
"path": "api/test/data/feed/api.prprpr.me",
"chars": 0,
"preview": ""
},
{
"path": "api/test/data/feed/apublica.org",
"chars": 219641,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\nxmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\nxmlns:"
},
{
"path": "api/test/data/feed/audiworld.com",
"chars": 29612,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
},
{
"path": "api/test/data/feed/boingboing",
"chars": 132664,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txml"
},
{
"path": "api/test/data/feed/bookshadow.com",
"chars": 62282,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss xmlns:atom=\"http://www.w3.org/2005/Atom\" version=\"2.0\"><channel><title>书影 - "
},
{
"path": "api/test/data/feed/dingxiaoyun555.blog.163.com",
"chars": 138143,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\r\n<rss version=\"2.0\" xmlns:slash=\"http://purl.org/rss/1.0/modules/slash/\" xmlns:d"
},
{
"path": "api/test/data/feed/django",
"chars": 28691,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\"><channel><title>The D"
},
{
"path": "api/test/data/feed/douban.com",
"chars": 6198,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\" xmlns"
},
{
"path": "api/test/data/feed/empty",
"chars": 0,
"preview": ""
},
{
"path": "api/test/data/feed/geektopia.es",
"chars": 17289,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss version=\"2.0\"\n xmlns:content=\"http://purl.org/rss/1."
},
{
"path": "api/test/data/feed/habr",
"chars": 32350,
"preview": "\n<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rss version=\"2.0\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" >\n <channel>\n "
},
{
"path": "api/test/data/feed/hackernews",
"chars": 12047,
"preview": "<rss version=\"2.0\"><channel><title>Hacker News</title><link>https://news.ycombinator.com/</link><description>Links for t"
},
{
"path": "api/test/data/feed/hackernoon-daily-dev",
"chars": 87622,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:content=\"http://purl.org/rs"
},
{
"path": "api/test/data/feed/kaiak.tw",
"chars": 768,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
},
{
"path": "api/test/data/feed/kottke",
"chars": 112327,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<feed xmlns=\"http://www.w3.org/2005/Atom\">\n <title>kottke.org</title>\n <lin"
},
{
"path": "api/test/data/feed/lemonde",
"chars": 17806,
"preview": "\n<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<rss version=\"2.0\" xmlns:atom=\"https://www.w3.org/2005/Atom\">\n <channel>\n <t"
},
{
"path": "api/test/data/feed/lobsters",
"chars": 18754,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\">\n <channel>\n <tit"
},
{
"path": "api/test/data/feed/lowendbox.com",
"chars": 122532,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
},
{
"path": "api/test/data/feed/malformed-hackernews",
"chars": 6849,
"preview": "<rss version=\"4.20\" encoding=\"ISO-8859-1\"><channel><title>Hacker News</title><link>htpt://:s://news.ycombinator.com/</li"
},
{
"path": "api/test/data/feed/maxwell-land-surveying.com",
"chars": 5590,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmln"
}
]
// ... and 245 more files (download for full content)
About this extraction
This page contains the full source code of the GetStream/Winds GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 445 files (7.5 MB), approximately 2.0M tokens, and a symbol index with 466 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.