Repository: mozilla/readability Branch: main Commit: 08be6b4bdb20 Files: 415 Total size: 27.1 MB Directory structure: gitextract_3shkyv0s/ ├── .gitattributes ├── .gitignore ├── .npmignore ├── .prettierrc.js ├── .release-it.json ├── .taskcluster.yml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── JSDOMParser.js ├── LICENSE.md ├── README.md ├── Readability-readerable.js ├── Readability.js ├── SECURITY.md ├── eslint.config.mjs ├── index.d.ts ├── index.js ├── package.json └── test/ ├── debug-testcase.js ├── generate-testcase.js ├── test-isProbablyReaderable.js ├── test-jsdomparser.js ├── test-pages/ │ ├── 001/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 002/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 003-metadata-preferred/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 004-metadata-space-separated-properties/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 005-unescape-html-entities/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── aclu/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── aktualne/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── archive-of-our-own/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ars-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── article-author-tag/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── base-url/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── base-url-base-element/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── base-url-base-element-relative/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── basic-tags-cleaning/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── bbc-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── blogger/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── breitbart/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── bug-1255978/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── buzzfeed-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── citylab-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── clean-links/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── cnet/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── cnet-svg-classes/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── cnn/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── comment-inside-script-parsing/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── daringfireball-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── data-url-image/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── dev418/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── dropbox-blog/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ebb-org/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ehow-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ehow-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── embedded-videos/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── engadget/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── firefox-nightly-blog/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── folha/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── gitlab-blog/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── gmw/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── google-sre-book-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── guardian-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── heise/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── herald-sun-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── hidden-nodes/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── hukumusume/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── iab-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ietf-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── invalid-attributes/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── js-link-replacement/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── keep-images/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── keep-tabular-data/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── la-nacion/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lazy-image-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lazy-image-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lazy-image-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lemonde-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── liberation-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lifehacker-post-comment-load/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lifehacker-working/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── links-in-tables/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lwn-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── mathjax/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── medicalnewstoday/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── medium-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── medium-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── medium-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── mercurial/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── metadata-content-missing/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── missing-paragraphs/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── mozilla-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── mozilla-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── msn/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── normalize-spaces/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── nytimes-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── nytimes-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── nytimes-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── nytimes-4/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── nytimes-5/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ol/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── parsely-metadata/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── pixnet/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── qq/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── quanta-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-aria-hidden/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-extra-brs/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-extra-paragraphs/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-script-tags/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── reordering-paragraphs/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── replace-brs/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── replace-font-tags/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── royal-road/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── rtl-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── rtl-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── rtl-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── rtl-4/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── salon-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── schema-org-context-object/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── seattletimes-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── simplyfound-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── social-buttons/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── spiceworks/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── style-tags-removal/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── svg-parsing/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── table-style-attributes/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── telegraph/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── theverge/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── title-and-h1-discrepancy/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── title-en-dash/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── tmz-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── toc-missing/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── topicseed-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── tumblr/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── v8-blog/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── videos-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── videos-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── visibility-hidden/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wapo-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wapo-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── webmd-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── webmd-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wikia/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wikipedia/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wikipedia-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wikipedia-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wikipedia-4/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── wordpress/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── yahoo-1/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── yahoo-2/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── yahoo-3/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── yahoo-4/ │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ └── youth/ │ ├── expected-metadata.json │ ├── expected.html │ └── source.html ├── test-readability.js └── utils.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ **/*.html linguist-detectable=false * text=auto eol=lf ================================================ FILE: .gitignore ================================================ .DS_Store npm-debug.log node_modules .metadata *.pyc *~ .*.sw? .sw? *.jar *.xpi ================================================ FILE: .npmignore ================================================ /benchmarks/ /test/ .gitattributes .release-it.json .taskcluster.yml ================================================ FILE: .prettierrc.js ================================================ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ /* eslint-env node */ module.exports = { arrowParens: "avoid", endOfLine: "lf", printWidth: 80, tabWidth: 2, trailingComma: "es5", }; ================================================ FILE: .release-it.json ================================================ { "plugins": { "@release-it/keep-a-changelog": { "addUnreleased": true, "filename": "CHANGELOG.md" } }, "github": { "release": false }, "git": { "requireBranch": "main", "pushRepo": "https://github.com/mozilla/readability.git" } } ================================================ FILE: .taskcluster.yml ================================================ version: 1 policy: pullRequests: public tasks: $let: head_rev: $if: 'tasks_for == "github-pull-request"' then: ${event.pull_request.head.sha} else: ${event.after} repository: $if: 'tasks_for == "github-pull-request"' then: ${event.pull_request.head.repo.html_url} else: ${event.repository.html_url} in: $match: '(tasks_for == "github-pull-request" && event["action"] in ["opened","reopened","synchronize"]) || (tasks_for == "github-push")': taskId: $eval: as_slugid("pr_task") provisionerId: proj-misc workerType: ci deadline: {$fromNow: '1 day'} payload: maxRunTime: 600 image: node command: - /bin/bash - '--login' - '-c' - >- git clone ${repository} repo && cd repo && git config advice.detachedHead false && git checkout ${head_rev} && npm install . && npm run lint && npm test metadata: name: Run tests and linter description: '' owner: '${event.sender.login}@users.noreply.github.com' source: '${event.repository.url}' ================================================ FILE: CHANGELOG.md ================================================ # Changelog Notable changes to readability will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project attempts to adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). For the purposes of Semantic Versioning, the readability output object for a given input document is not considered a stable API. That is, minor version increments may change this output. Patch version increments will only do so in ways that are strict improvements (e.g. from empty strings or exceptions to something more reasonable). ## [Unreleased] ## [0.6.0] - 2025-03-03 - [Add Parsely tags as a fallback metadata source](https://github.com/mozilla/readability/pull/865) - [Fix the case that jsonld parse process is ignored when context url include the trailing slash](https://github.com/mozilla/readability/pull/833) - [Improve data table support](https://github.com/mozilla/readability/pull/858) - [Fixed situations where short paragraphs of legitimate content would be excluded](https://github.com/mozilla/readability/pull/867) - [Add an option to modify link density value](https://github.com/mozilla/readability/pull/874) - [Byline metadata should lead to not deleting lookalike non-byline content](https://github.com/mozilla/readability/pull/869) - [Avoid removing headers on gitlab](https://github.com/mozilla/readability/pull/885) - [Improved HTML character unescaping](https://github.com/mozilla/readability/pull/896) - Various performance improvements: [#894](https://github.com/mozilla/readability/pull/894), [#892](https://github.com/mozilla/readability/pull/892), [#893](https://github.com/mozilla/readability/pull/893), [#915](https://github.com/mozilla/readability/pull/915), - [Fix broken JSONLD context handling](https://github.com/mozilla/readability/pull/902) - [Include Jekyll footnotes in output](https://github.com/mozilla/readability/pull/907) - [Handle schema.org context objects](https://github.com/mozilla/readability/pull/940) - [Fix invalid attributes breaking parsing](https://github.com/mozilla/readability/pull/918) - [Include article:author metadata](https://github.com/mozilla/readability/pull/942) - [Handle itemprop=name for author metadata](https://github.com/mozilla/readability/pull/943) - [Improve typescript definitions](https://github.com/mozilla/readability/pull/944) - [Handle JSONLD Arrays](https://github.com/mozilla/readability/pull/947) ## [0.5.0] - 2023-12-15 - [Add published time metadata](https://github.com/mozilla/readability/pull/813) - [Expanded comma detection to non-Latin commas](https://github.com/mozilla/readability/pull/796) - [Fix detection of elements hidden with style="visibility: hidden"](https://github.com/mozilla/readability/pull/817) ## [0.4.4] - 2023-03-31 - Fixed [undefined `li_count` variable breaking use of readability in Cloudflare workers](https://github.com/mozilla/readability/issues/791) ## [0.4.3] - 2023-03-22 - Fixed [`aria-modal` cookie dialogs interfering with readability](https://github.com/mozilla/readability/pull/746) - Fixed [lists of images not showing](https://github.com/mozilla/readability/pull/738) - [Updated type information for TypeScript](https://github.com/mozilla/readability/pull/734) - [Simplify `script` and `noscript` removal](https://github.com/mozilla/readability/pull/762) - [Updated dependencies](https://github.com/mozilla/readability/pull/770) - [Added allowedVideoRegex option to override the default](https://github.com/mozilla/readability/pull/788) ## [0.4.2] - 2022-02-09 - Fix [compatibility with DOM implementations where the `childNodes` property is not live](https://github.com/mozilla/readability/pull/694) ([x2](https://github.com/mozilla/readability/pull/677)). - Lazily-loaded image references [will no longer use the `alt` attribute](https://github.com/mozilla/readability/pull/689) to find images. - `parse()` [provides the root element's `lang` attribute](https://github.com/mozilla/readability/pull/721) - `isProbablyReadable` [includes article tags](https://github.com/mozilla/readability/pull/724) - Improvements to JSON-LD support - [Continue parsing other JSON-LD elements until we find one we can support](https://github.com/mozilla/readability/pull/713) - [Prefer using headline for article title](https://github.com/mozilla/readability/pull/713) ## [0.4.1] - 2021-01-13 ### Added - Typescript type definition file (`.d.ts`). ## [0.4.0] - 2020-12-23 ### Added - `isProbablyReaderable` [can now take an optional options object](https://github.com/mozilla/readability/pull/634) to configure it, allowing you to specify the minimum content length, minimum score, and how to check if nodes are visible. - Better support for [deeply-nested content](https://github.com/mozilla/readability/pull/611). - Readability is now more likely to [keep tables of content](https://github.com/mozilla/readability/pull/646). - Better support for [content in `` tags](https://github.com/mozilla/readability/pull/647). - Readability (finally) no longer [throws away all `

` tags](https://github.com/mozilla/readability/pull/650). ### Changed - JSON-LD [support for multiple authors](https://github.com/mozilla/readability/pull/618) was improved. - Elements with roles `menu`, `menubar`, `complementary`, `navigation`, `alert`, `alertdialog`, `dialog` will [all be removed](https://github.com/mozilla/readability/pull/619). ## [0.3.0] - 2020-08-05 The first version that was published on NPM. Previously, we did not consistently version anything, nor did we publish to NPM. At some point, we may wish to expand this changelog into the past. ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Community Participation Guidelines This repository is governed by Mozilla's code of conduct and etiquette guidelines. For more details, please read the [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). ## How to Report For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Thank you for wanting to help make `readability` better! For outstanding issues, see the issue list in this repo, as well as this [bug list](https://bugzilla.mozilla.org/buglist.cgi?component=Reader%20Mode&product=Toolkit&bug_status=__open__&limit=0). Any changes to the main code should be reviewed by an [appropriate Firefox/toolkit peer](https://wiki.mozilla.org/Modules/Firefox), such as [@gijsk](https://github.com/gijsk), since these changes will be merged to mozilla-central and shipped in Firefox. To test local changes to Readability.js, you can use the [automated tests](#tests). This repository is governed by Mozilla's code of conduct and etiquette guidelines. For more details, please read the [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). ## Tests [![Build Status](https://community-tc.services.mozilla.com/api/github/v1/repository/mozilla/readability/main/badge.svg)](https://community-tc.services.mozilla.com/api/github/v1/repository/mozilla/readability/main/latest) Please run [eslint](http://eslint.org/) as a first check that your changes are valid JS and adhere to our style guidelines: $ npm run lint To run the test suite: $ npm test To run a specific test page by its name: $ npm test -- -g 001 To run the test suite in TDD mode: $ npm test -- -w Combo time: $ npm test -- -w -g 001 ### Add new tests There's a [node script](https://github.com/mozilla/readability/blob/master/test/generate-testcase.js) to help you create new tests. You can run it using: $ node test/generate-testcase.js slug https://example.com/article Replacing `slug` with the identifier the test should use, and providing a URL to an actual article on which the test should be based. If your test case involves dynamic content, you can save the page to disk and pass the file as a `file://` URL. You may need to make the `tidy` binary executable before that script will succeed. If you see an `EACCES` error when running that script, try: # On MacOS $ chmod +x ./node_modules/htmltidy2/bin/darwin/tidy # On Linux, the linuxXX may change depending on your platform # The specific path may change depending on the node tools you use $ chmod +x ./node_modules/htmltidy2/bin/linux64/tidy ## Pull Requests We're always happy to see pull requests to improve readability. Please ensure you run the linter and [tests](#tests) before submitting a PR. If you're changing the algorithm to fix a specific page/article, please [add new tests](#add-new-tests) for the case you're fixing, so we avoid breaking it in future. ## Steps to release 1. Ensure [CHANGELOG.md](CHANGELOG.md) is up-to-date. ``git log `npm view . version`...master `` may help with this. 2. Run `npm run release` to create a release, which should: 1. `npm version [patch | minor | major]`, depending on the nature of the changes according to [semver](https://semver.org/). This will bump the version in `package.json` and `package-lock.json` and create a commit and Git tag for the release. 2. `npm publish` to push the release to the npm registry. 3. `git push origin head --follow-tags` to push the new commit and tag to GitHub. ## Keeping a changelog Ensure significant changes are added to `CHANGELOG.md`. Do not add changes that only affect tests or documentation. ================================================ FILE: JSDOMParser.js ================================================ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ /** * This is a relatively lightweight DOMParser that is safe to use in a web * worker. This is far from a complete DOM implementation; however, it should * contain the minimal set of functionality necessary for Readability.js. * * Aside from not implementing the full DOM API, there are other quirks to be * aware of when using the JSDOMParser: * * 1) Properly formed HTML/XML must be used. This means you should be extra * careful when using this parser on anything received directly from an * XMLHttpRequest. Providing a serialized string from an XMLSerializer, * however, should be safe (since the browser's XMLSerializer should * generate valid HTML/XML). Therefore, if parsing a document from an XHR, * the recommended approach is to do the XHR in the main thread, use * XMLSerializer.serializeToString() on the responseXML, and pass the * resulting string to the worker. * * 2) Live NodeLists are not supported. DOM methods and properties such as * getElementsByTagName() and childNodes return standard arrays. If you * want these lists to be updated when nodes are removed or added to the * document, you must take care to manually update them yourself. */ (function (global) { // XML only defines these and the numeric ones: var entityTable = { lt: "<", gt: ">", amp: "&", quot: '"', apos: "'", }; var reverseEntityTable = { "<": "<", ">": ">", "&": "&", '"': """, "'": "'", }; function encodeTextContentHTML(s) { return s.replace(/[&<>]/g, function (x) { return reverseEntityTable[x]; }); } function encodeHTML(s) { return s.replace(/[&<>'"]/g, function (x) { return reverseEntityTable[x]; }); } function decodeHTML(str) { return str .replace(/&(quot|amp|apos|lt|gt);/g, function (match, tag) { return entityTable[tag]; }) .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function (match, hex, numStr) { var num = parseInt(hex || numStr, hex ? 16 : 10); // these character references are replaced by a conforming HTML parser if (num == 0 || num > 0x10ffff || (num >= 0xd800 && num <= 0xdfff)) { num = 0xfffd; } return String.fromCodePoint(num); }); } // When a style is set in JS, map it to the corresponding CSS attribute var styleMap = { alignmentBaseline: "alignment-baseline", background: "background", backgroundAttachment: "background-attachment", backgroundClip: "background-clip", backgroundColor: "background-color", backgroundImage: "background-image", backgroundOrigin: "background-origin", backgroundPosition: "background-position", backgroundPositionX: "background-position-x", backgroundPositionY: "background-position-y", backgroundRepeat: "background-repeat", backgroundRepeatX: "background-repeat-x", backgroundRepeatY: "background-repeat-y", backgroundSize: "background-size", baselineShift: "baseline-shift", border: "border", borderBottom: "border-bottom", borderBottomColor: "border-bottom-color", borderBottomLeftRadius: "border-bottom-left-radius", borderBottomRightRadius: "border-bottom-right-radius", borderBottomStyle: "border-bottom-style", borderBottomWidth: "border-bottom-width", borderCollapse: "border-collapse", borderColor: "border-color", borderImage: "border-image", borderImageOutset: "border-image-outset", borderImageRepeat: "border-image-repeat", borderImageSlice: "border-image-slice", borderImageSource: "border-image-source", borderImageWidth: "border-image-width", borderLeft: "border-left", borderLeftColor: "border-left-color", borderLeftStyle: "border-left-style", borderLeftWidth: "border-left-width", borderRadius: "border-radius", borderRight: "border-right", borderRightColor: "border-right-color", borderRightStyle: "border-right-style", borderRightWidth: "border-right-width", borderSpacing: "border-spacing", borderStyle: "border-style", borderTop: "border-top", borderTopColor: "border-top-color", borderTopLeftRadius: "border-top-left-radius", borderTopRightRadius: "border-top-right-radius", borderTopStyle: "border-top-style", borderTopWidth: "border-top-width", borderWidth: "border-width", bottom: "bottom", boxShadow: "box-shadow", boxSizing: "box-sizing", captionSide: "caption-side", clear: "clear", clip: "clip", clipPath: "clip-path", clipRule: "clip-rule", color: "color", colorInterpolation: "color-interpolation", colorInterpolationFilters: "color-interpolation-filters", colorProfile: "color-profile", colorRendering: "color-rendering", content: "content", counterIncrement: "counter-increment", counterReset: "counter-reset", cursor: "cursor", direction: "direction", display: "display", dominantBaseline: "dominant-baseline", emptyCells: "empty-cells", enableBackground: "enable-background", fill: "fill", fillOpacity: "fill-opacity", fillRule: "fill-rule", filter: "filter", cssFloat: "float", floodColor: "flood-color", floodOpacity: "flood-opacity", font: "font", fontFamily: "font-family", fontSize: "font-size", fontStretch: "font-stretch", fontStyle: "font-style", fontVariant: "font-variant", fontWeight: "font-weight", glyphOrientationHorizontal: "glyph-orientation-horizontal", glyphOrientationVertical: "glyph-orientation-vertical", height: "height", imageRendering: "image-rendering", kerning: "kerning", left: "left", letterSpacing: "letter-spacing", lightingColor: "lighting-color", lineHeight: "line-height", listStyle: "list-style", listStyleImage: "list-style-image", listStylePosition: "list-style-position", listStyleType: "list-style-type", margin: "margin", marginBottom: "margin-bottom", marginLeft: "margin-left", marginRight: "margin-right", marginTop: "margin-top", marker: "marker", markerEnd: "marker-end", markerMid: "marker-mid", markerStart: "marker-start", mask: "mask", maxHeight: "max-height", maxWidth: "max-width", minHeight: "min-height", minWidth: "min-width", opacity: "opacity", orphans: "orphans", outline: "outline", outlineColor: "outline-color", outlineOffset: "outline-offset", outlineStyle: "outline-style", outlineWidth: "outline-width", overflow: "overflow", overflowX: "overflow-x", overflowY: "overflow-y", padding: "padding", paddingBottom: "padding-bottom", paddingLeft: "padding-left", paddingRight: "padding-right", paddingTop: "padding-top", page: "page", pageBreakAfter: "page-break-after", pageBreakBefore: "page-break-before", pageBreakInside: "page-break-inside", pointerEvents: "pointer-events", position: "position", quotes: "quotes", resize: "resize", right: "right", shapeRendering: "shape-rendering", size: "size", speak: "speak", src: "src", stopColor: "stop-color", stopOpacity: "stop-opacity", stroke: "stroke", strokeDasharray: "stroke-dasharray", strokeDashoffset: "stroke-dashoffset", strokeLinecap: "stroke-linecap", strokeLinejoin: "stroke-linejoin", strokeMiterlimit: "stroke-miterlimit", strokeOpacity: "stroke-opacity", strokeWidth: "stroke-width", tableLayout: "table-layout", textAlign: "text-align", textAnchor: "text-anchor", textDecoration: "text-decoration", textIndent: "text-indent", textLineThrough: "text-line-through", textLineThroughColor: "text-line-through-color", textLineThroughMode: "text-line-through-mode", textLineThroughStyle: "text-line-through-style", textLineThroughWidth: "text-line-through-width", textOverflow: "text-overflow", textOverline: "text-overline", textOverlineColor: "text-overline-color", textOverlineMode: "text-overline-mode", textOverlineStyle: "text-overline-style", textOverlineWidth: "text-overline-width", textRendering: "text-rendering", textShadow: "text-shadow", textTransform: "text-transform", textUnderline: "text-underline", textUnderlineColor: "text-underline-color", textUnderlineMode: "text-underline-mode", textUnderlineStyle: "text-underline-style", textUnderlineWidth: "text-underline-width", top: "top", unicodeBidi: "unicode-bidi", unicodeRange: "unicode-range", vectorEffect: "vector-effect", verticalAlign: "vertical-align", visibility: "visibility", whiteSpace: "white-space", widows: "widows", width: "width", wordBreak: "word-break", wordSpacing: "word-spacing", wordWrap: "word-wrap", writingMode: "writing-mode", zIndex: "z-index", zoom: "zoom", }; // Elements that can be self-closing var voidElems = { area: true, base: true, br: true, col: true, command: true, embed: true, hr: true, img: true, input: true, link: true, meta: true, param: true, source: true, wbr: true, }; var whitespace = [" ", "\t", "\n", "\r"]; // See https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType var nodeTypes = { ELEMENT_NODE: 1, ATTRIBUTE_NODE: 2, TEXT_NODE: 3, CDATA_SECTION_NODE: 4, ENTITY_REFERENCE_NODE: 5, ENTITY_NODE: 6, PROCESSING_INSTRUCTION_NODE: 7, COMMENT_NODE: 8, DOCUMENT_NODE: 9, DOCUMENT_TYPE_NODE: 10, DOCUMENT_FRAGMENT_NODE: 11, NOTATION_NODE: 12, }; function getElementsByTagName(tag) { tag = tag.toUpperCase(); var elems = []; var allTags = tag === "*"; function getElems(node) { var length = node.children.length; for (var i = 0; i < length; i++) { var child = node.children[i]; if (allTags || child.tagName === tag) { elems.push(child); } getElems(child); } } getElems(this); elems._isLiveNodeList = true; return elems; } var Node = function () {}; Node.prototype = { attributes: null, childNodes: null, localName: null, nodeName: null, parentNode: null, textContent: null, nextSibling: null, previousSibling: null, get firstChild() { return this.childNodes[0] || null; }, get firstElementChild() { return this.children[0] || null; }, get lastChild() { return this.childNodes[this.childNodes.length - 1] || null; }, get lastElementChild() { return this.children[this.children.length - 1] || null; }, /** * The workhorse for all node insertion operations. The public methods * (`appendChild()`, `insertBefore()`, `replaceChild()`) are thin wrappers * around this. * * @private * @param {Node[]} nodes - An array of nodes to insert. It is assumed that * these nodes are distinct, and are not children of this object. * @param {Number} index - A valid index to insert `nodes` at, or -1 to * indicate insertion as the last children. * @returns {void} */ _insertNodesAtIndex(nodes, index) { if (!nodes.length) { return; } // Detach nodes from their previous parents. for (var i = 0; i < nodes.length; i++) { if (nodes[i].parentNode) { nodes[i].remove(); } } var afterSibling = index === -1 ? null : this.childNodes[index]; // Store the previous sibling before we modify the DOM. var prevSibling = afterSibling ? afterSibling.previousSibling : this.lastChild; // Insert nodes into childNodes. var insertionPoint = index === -1 ? this.childNodes.length : index; Array.prototype.splice.apply( this.childNodes, [insertionPoint, 0].concat(nodes) ); // Update parentNode and sibling pointers for the new nodes. for (var j = 0; j < nodes.length; j++) { var node = nodes[j]; node.parentNode = this; node.previousSibling = prevSibling; if (prevSibling) { prevSibling.nextSibling = node; } prevSibling = node; } var lastInsertedNode = nodes[nodes.length - 1]; lastInsertedNode.nextSibling = afterSibling; if (afterSibling) { afterSibling.previousSibling = lastInsertedNode; } // Filter for element nodes and update children array and pointers. var elementsToInsert = []; for (var k = 0; k < nodes.length; k++) { if (nodes[k].nodeType === Node.ELEMENT_NODE) { elementsToInsert.push(nodes[k]); } } if (elementsToInsert.length) { // Find the next element sibling to use as an insertion reference. // This is done after `childNodes` is modified, as the forward // traversal from `afterSibling` remains valid. var afterElem = afterSibling; while (afterElem && afterElem.nodeType !== Node.ELEMENT_NODE) { afterElem = afterElem.nextSibling; } // Store the previous element sibling before more DOM modifications. var prevElem = afterElem ? afterElem.previousElementSibling : this.lastElementChild; var afterElemIndex = afterElem ? this.children.indexOf(afterElem) : -1; var elemInsertionPoint = afterElemIndex === -1 ? this.children.length : afterElemIndex; Array.prototype.splice.apply( this.children, [elemInsertionPoint, 0].concat(elementsToInsert) ); for (var l = 0; l < elementsToInsert.length; l++) { var elem = elementsToInsert[l]; elem.previousElementSibling = prevElem; if (prevElem) { prevElem.nextElementSibling = elem; } prevElem = elem; } var lastInsertedElem = elementsToInsert[elementsToInsert.length - 1]; lastInsertedElem.nextElementSibling = afterElem; if (afterElem) { afterElem.previousElementSibling = lastInsertedElem; } } }, appendChild(child) { var nodes = child.nodeType === Node.DOCUMENT_FRAGMENT_NODE ? Array.from(child.childNodes) : [child]; this._insertNodesAtIndex(nodes, -1); return child; }, insertBefore(newNode, referenceNode) { if (newNode === referenceNode) { return newNode; } var nodes = newNode.nodeType === Node.DOCUMENT_FRAGMENT_NODE ? Array.from(newNode.childNodes) : [newNode]; var index = referenceNode ? this.childNodes.indexOf(referenceNode) : -1; if (referenceNode && index === -1) { throw new Error("insertBefore: reference node not found"); } this._insertNodesAtIndex(nodes, index); return newNode; }, remove() { let parent = this.parentNode; if (!parent) { // We were already detached so there's nothing to do. return this; } var childNodes = parent.childNodes; var childIndex = childNodes.indexOf(this); if (childIndex === -1) { throw new Error("removeChild: node not found"); } this.parentNode = null; var prev = this.previousSibling; var next = this.nextSibling; if (prev) { prev.nextSibling = next; } if (next) { next.previousSibling = prev; } childNodes.splice(childIndex, 1); if (this.nodeType === Node.ELEMENT_NODE) { var prevElem = this.previousElementSibling; var nextElem = this.nextElementSibling; if (prevElem) { prevElem.nextElementSibling = nextElem; } if (nextElem) { nextElem.previousElementSibling = prevElem; } parent.children.splice(parent.children.indexOf(this), 1); this.previousElementSibling = this.nextElementSibling = null; } this.previousSibling = this.nextSibling = null; return this; }, removeChild(child) { return child.remove(); }, replaceChild(newNode, oldNode) { if (newNode === oldNode) { return oldNode; } if (oldNode.parentNode !== this) { throw new Error( "replaceChild: node to be replaced is not a child of this node" ); } // Insert the new node(s) before the node to be replaced. this.insertBefore(newNode, oldNode); // Now, remove the old node. oldNode.remove(); return oldNode; }, __JSDOMParser__: true, }; for (var nodeType in nodeTypes) { Node[nodeType] = Node.prototype[nodeType] = nodeTypes[nodeType]; } var Attribute = function (name, value) { this.name = name; this._value = value; }; Attribute.prototype = { get value() { return this._value; }, setValue(newValue) { this._value = newValue; }, getEncodedValue() { return encodeHTML(this._value); }, // Cheat horribly. This is fine for our usecases. cloneNode() { return this; }, }; var Comment = function () { this.childNodes = []; }; Comment.prototype = { __proto__: Node.prototype, nodeName: "#comment", nodeType: Node.COMMENT_NODE, }; var DocumentFragment = function () { this.childNodes = []; this.children = []; }; DocumentFragment.prototype = { __proto__: Node.prototype, nodeName: "#document-fragment", nodeType: Node.DOCUMENT_FRAGMENT_NODE, }; var Text = function () { this.childNodes = []; }; Text.prototype = { __proto__: Node.prototype, nodeName: "#text", nodeType: Node.TEXT_NODE, get textContent() { if (typeof this._textContent === "undefined") { this._textContent = decodeHTML(this._innerHTML || ""); } return this._textContent; }, get innerHTML() { if (typeof this._innerHTML === "undefined") { this._innerHTML = encodeTextContentHTML(this._textContent || ""); } return this._innerHTML; }, set innerHTML(newHTML) { this._innerHTML = newHTML; delete this._textContent; }, set textContent(newText) { this._textContent = newText; delete this._innerHTML; }, }; var Document = function (url) { this.documentURI = url; this.styleSheets = []; this.childNodes = []; this.children = []; }; Document.prototype = { __proto__: Node.prototype, nodeName: "#document", nodeType: Node.DOCUMENT_NODE, title: "", getElementsByTagName, getElementById(id) { function getElem(node) { var length = node.children.length; if (node.id === id) { return node; } for (var i = 0; i < length; i++) { var el = getElem(node.children[i]); if (el) { return el; } } return null; } return getElem(this); }, createElement(tag) { var node = new Element(tag); return node; }, createTextNode(text) { var node = new Text(); node.textContent = text; return node; }, createDocumentFragment() { return new DocumentFragment(); }, get baseURI() { if (!this.hasOwnProperty("_baseURI")) { this._baseURI = this.documentURI; var baseElements = this.getElementsByTagName("base"); var href = baseElements[0] && baseElements[0].getAttribute("href"); if (href) { try { this._baseURI = new URL(href, this._baseURI).href; } catch (ex) { /* Just fall back to documentURI */ } } } return this._baseURI; }, }; var Element = function (tag) { // We use this to find the closing tag. this._matchingTag = tag; // We're explicitly a non-namespace aware parser, we just pretend it's all HTML. var lastColonIndex = tag.lastIndexOf(":"); if (lastColonIndex != -1) { tag = tag.substring(lastColonIndex + 1); } this.attributes = []; this.childNodes = []; this.children = []; this.nextElementSibling = this.previousElementSibling = null; this.localName = tag.toLowerCase(); this.tagName = tag.toUpperCase(); this.style = new Style(this); }; Element.prototype = { __proto__: Node.prototype, nodeType: Node.ELEMENT_NODE, getElementsByTagName, get className() { return this.getAttribute("class") || ""; }, set className(str) { this.setAttribute("class", str); }, get id() { return this.getAttribute("id") || ""; }, set id(str) { this.setAttribute("id", str); }, get href() { return this.getAttribute("href") || ""; }, set href(str) { this.setAttribute("href", str); }, get src() { return this.getAttribute("src") || ""; }, set src(str) { this.setAttribute("src", str); }, get srcset() { return this.getAttribute("srcset") || ""; }, set srcset(str) { this.setAttribute("srcset", str); }, get nodeName() { return this.tagName; }, get innerHTML() { function getHTML(node) { var i = 0; for (i = 0; i < node.childNodes.length; i++) { var child = node.childNodes[i]; if (child.localName) { arr.push("<" + child.localName); // serialize attribute list for (var j = 0; j < child.attributes.length; j++) { var attr = child.attributes[j]; // the attribute value will be HTML escaped. var val = attr.getEncodedValue(); var quote = !val.includes('"') ? '"' : "'"; arr.push(" " + attr.name + "=" + quote + val + quote); } if (child.localName in voidElems && !child.childNodes.length) { // if this is a self-closing element, end it here arr.push("/>"); } else { // otherwise, add its children arr.push(">"); getHTML(child); arr.push(""); } } else { // This is a text node, so asking for innerHTML won't recurse. arr.push(child.innerHTML); } } } // Using Array.join() avoids the overhead from lazy string concatenation. var arr = []; getHTML(this); return arr.join(""); }, set innerHTML(html) { var parser = new JSDOMParser(); var node = parser.parse(html); var i; for (i = this.childNodes.length; --i >= 0; ) { this.childNodes[i].parentNode = null; } this.childNodes = node.childNodes; this.children = node.children; for (i = this.childNodes.length; --i >= 0; ) { this.childNodes[i].parentNode = this; } }, set textContent(text) { // clear parentNodes for existing children for (var i = this.childNodes.length; --i >= 0; ) { this.childNodes[i].parentNode = null; } var node = new Text(); this.childNodes = [node]; this.children = []; node.textContent = text; node.parentNode = this; }, get textContent() { function getText(node) { var nodes = node.childNodes; for (var i = 0; i < nodes.length; i++) { var child = nodes[i]; if (child.nodeType === 3) { text.push(child.textContent); } else { getText(child); } } } // Using Array.join() avoids the overhead from lazy string concatenation. // See http://blog.cdleary.com/2012/01/string-representation-in-spidermonkey/#ropes var text = []; getText(this); return text.join(""); }, getAttribute(name) { for (var i = this.attributes.length; --i >= 0; ) { var attr = this.attributes[i]; if (attr.name === name) { return attr.value; } } return undefined; }, setAttribute(name, value) { for (var i = this.attributes.length; --i >= 0; ) { var attr = this.attributes[i]; if (attr.name === name) { attr.setValue(value); return; } } this.attributes.push(new Attribute(name, value)); }, setAttributeNode(node) { this.setAttribute(node.name, node.value); }, removeAttribute(name) { for (var i = this.attributes.length; --i >= 0; ) { var attr = this.attributes[i]; if (attr.name === name) { this.attributes.splice(i, 1); break; } } }, hasAttribute(name) { return this.attributes.some(function (attr) { return attr.name == name; }); }, }; var Style = function (node) { this.node = node; }; // getStyle() and setStyle() use the style attribute string directly. This // won't be very efficient if there are a lot of style manipulations, but // it's the easiest way to make sure the style attribute string and the JS // style property stay in sync. Readability.js doesn't do many style // manipulations, so this should be okay. Style.prototype = { getStyle(styleName) { var attr = this.node.getAttribute("style"); if (!attr) { return undefined; } var styles = attr.split(";"); for (var i = 0; i < styles.length; i++) { var style = styles[i].split(":"); var name = style[0].trim(); if (name === styleName) { return style[1].trim(); } } return undefined; }, setStyle(styleName, styleValue) { var value = this.node.getAttribute("style") || ""; var index = 0; do { var next = value.indexOf(";", index) + 1; var length = next - index - 1; var style = length > 0 ? value.substr(index, length) : value.substr(index); if (style.substr(0, style.indexOf(":")).trim() === styleName) { value = value.substr(0, index).trim() + (next ? " " + value.substr(next).trim() : ""); break; } index = next; } while (index); value += " " + styleName + ": " + styleValue + ";"; this.node.setAttribute("style", value.trim()); }, }; // For each item in styleMap, define a getter and setter on the style // property. for (var jsName in styleMap) { (function (cssName) { Style.prototype.__defineGetter__(jsName, function () { return this.getStyle(cssName); }); Style.prototype.__defineSetter__(jsName, function (value) { this.setStyle(cssName, value); }); })(styleMap[jsName]); } var JSDOMParser = function () { this.currentChar = 0; // In makeElementNode() we build up many strings one char at a time. Using // += for this results in lots of short-lived intermediate strings. It's // better to build an array of single-char strings and then join() them // together at the end. And reusing a single array (i.e. |this.strBuf|) // over and over for this purpose uses less memory than using a new array // for each string. this.strBuf = []; // Similarly, we reuse this array to return the two arguments from // makeElementNode(), which saves us from having to allocate a new array // every time. this.retPair = []; this.errorState = ""; }; JSDOMParser.prototype = { error(m) { if (typeof console !== "undefined") { // eslint-disable-next-line no-console console.log("JSDOMParser error: " + m + "\n"); } else if (typeof dump !== "undefined") { /* global dump */ dump("JSDOMParser error: " + m + "\n"); } this.errorState += m + "\n"; }, /** * Look at the next character without advancing the index. */ peekNext() { return this.html[this.currentChar]; }, /** * Get the next character and advance the index. */ nextChar() { return this.html[this.currentChar++]; }, /** * Called after a quote character is read. This finds the next quote * character and returns the text string in between. */ readString(quote) { var str; var n = this.html.indexOf(quote, this.currentChar); if (n === -1) { this.currentChar = this.html.length; str = null; } else { str = this.html.substring(this.currentChar, n); this.currentChar = n + 1; } return str; }, /** * Called when parsing a node. This finds the next name/value attribute * pair and adds the result to the attributes list. */ readAttribute(node) { var name = ""; var n = this.html.indexOf("=", this.currentChar); if (n === -1) { this.currentChar = this.html.length; } else { // Read until a '=' character is hit; this will be the attribute key name = this.html.substring(this.currentChar, n); this.currentChar = n + 1; } if (!name) { return; } // After a '=', we should see a '"' for the attribute value var c = this.nextChar(); if (c !== '"' && c !== "'") { this.error("Error reading attribute " + name + ", expecting '\"'"); return; } // Read the attribute value (and consume the matching quote) var value = this.readString(c); node.attributes.push(new Attribute(name, decodeHTML(value))); }, /** * Parses and returns an Element node. This is called after a '<' has been * read. * * @returns an array; the first index of the array is the parsed node; * the second index is a boolean indicating whether this is a void * Element */ makeElementNode(retPair) { var c = this.nextChar(); // Read the Element tag name var strBuf = this.strBuf; strBuf.length = 0; while (!whitespace.includes(c) && c !== ">" && c !== "/") { if (c === undefined) { return false; } strBuf.push(c); c = this.nextChar(); } var tag = strBuf.join(""); if (!tag) { return false; } var node = new Element(tag); // Read Element attributes while (c !== "/" && c !== ">") { if (c === undefined) { return false; } while (whitespace.includes(this.html[this.currentChar++])) { // Advance cursor to first non-whitespace char. } this.currentChar--; c = this.nextChar(); if (c !== "/" && c !== ">") { --this.currentChar; this.readAttribute(node); } } // If this is a self-closing tag, read '/>' var closed = false; if (c === "/") { closed = true; c = this.nextChar(); if (c !== ">") { this.error("expected '>' to close " + tag); return false; } } retPair[0] = node; retPair[1] = closed; return true; }, /** * If the current input matches this string, advance the input index; * otherwise, do nothing. * * @returns whether input matched string */ match(str) { var strlen = str.length; if ( this.html.substr(this.currentChar, strlen).toLowerCase() === str.toLowerCase() ) { this.currentChar += strlen; return true; } return false; }, /** * Searches the input until a string is found and discards all input up to * and including the matched string. */ discardTo(str) { var index = this.html.indexOf(str, this.currentChar) + str.length; if (index === -1) { this.currentChar = this.html.length; } this.currentChar = index; }, /** * Reads child nodes for the given node. */ readChildren(node) { var child; while ((child = this.readNode())) { // Don't keep Comment nodes if (child.nodeType !== 8) { node.appendChild(child); } } }, discardNextComment() { if (this.match("--")) { this.discardTo("-->"); } else { var c = this.nextChar(); while (c !== ">") { if (c === undefined) { return null; } if (c === '"' || c === "'") { this.readString(c); } c = this.nextChar(); } } return new Comment(); }, /** * Reads the next child node from the input. If we're reading a closing * tag, or if we've reached the end of input, return null. * * @returns the node */ readNode() { var c = this.nextChar(); if (c === undefined) { return null; } // Read any text as Text node var textNode; if (c !== "<") { --this.currentChar; textNode = new Text(); var n = this.html.indexOf("<", this.currentChar); // We're not expecting XSS type exploitation inside JSDOMParser, // we just have to implement innerHTML stuff... /* eslint-disable no-unsanitized/property */ if (n === -1) { textNode.innerHTML = this.html.substring( this.currentChar, this.html.length ); this.currentChar = this.html.length; } else { textNode.innerHTML = this.html.substring(this.currentChar, n); this.currentChar = n; } /* eslint-enable no-unsanitized/property */ return textNode; } if (this.match("![CDATA[")) { var endChar = this.html.indexOf("]]>", this.currentChar); if (endChar === -1) { this.error("unclosed CDATA section"); return null; } textNode = new Text(); textNode.textContent = this.html.substring(this.currentChar, endChar); this.currentChar = endChar + "]]>".length; return textNode; } c = this.peekNext(); // Read Comment node. Normally, Comment nodes know their inner // textContent, but we don't really care about Comment nodes (we throw // them away in readChildren()). So just returning an empty Comment node // here is sufficient. if (c === "!" || c === "?") { // We're still before the ! or ? that is starting this comment: this.currentChar++; return this.discardNextComment(); } // If we're reading a closing tag, return null. This means we've reached // the end of this set of child nodes. if (c === "/") { --this.currentChar; return null; } // Otherwise, we're looking at an Element node var result = this.makeElementNode(this.retPair); if (!result) { return null; } var node = this.retPair[0]; var closed = this.retPair[1]; var localName = node.localName; // If this isn't a void Element, read its child nodes if (!closed) { this.readChildren(node); var closingTag = ""; if (!this.match(closingTag)) { this.error( "expected '" + closingTag + "' and got " + this.html.substr(this.currentChar, closingTag.length) ); return null; } } // Only use the first title, because SVG might have other // title elements which we don't care about (medium.com // does this, at least). if (localName === "title" && !this.doc.title) { this.doc.title = node.textContent.trim(); } else if (localName === "head") { this.doc.head = node; } else if (localName === "body") { this.doc.body = node; } else if (localName === "html") { this.doc.documentElement = node; } return node; }, /** * Parses an HTML string and returns a JS implementation of the Document. */ parse(html, url) { this.html = html; var doc = (this.doc = new Document(url)); this.readChildren(doc); // If this is an HTML document, remove root-level children except for the // node if (doc.documentElement) { for (var i = doc.childNodes.length; --i >= 0; ) { var child = doc.childNodes[i]; if (child !== doc.documentElement) { child.remove(); } } } return doc; }, }; // Attach the standard DOM types to the global scope global.Node = Node; global.Comment = Comment; global.Document = Document; global.DocumentFragment = DocumentFragment; global.Element = Element; global.Text = Text; // Attach JSDOMParser to the global scope global.JSDOMParser = JSDOMParser; })(this); if (typeof module === "object") { /* eslint-disable-next-line no-redeclare */ /* global module */ module.exports = this.JSDOMParser; } ================================================ FILE: LICENSE.md ================================================ Copyright (c) 2010 Arc90 Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # Readability.js A standalone version of the readability library used for [Firefox Reader View](https://support.mozilla.org/kb/firefox-reader-view-clutter-free-web-pages). ## Installation Readability is available on npm: ```bash npm install @mozilla/readability ``` You can then `require()` it, or for web-based projects, load the `Readability.js` script from your webpage. ## Basic usage To parse a document, you must create a new `Readability` object from a DOM document object, and then call the [`parse()`](#parse) method. Here's an example: ```javascript var article = new Readability(document).parse(); ``` If you use Readability in a web browser, you will likely be able to use a `document` reference from elsewhere (e.g. fetched via XMLHttpRequest, in a same-origin `

Policisté budou s hygieniky častěji kontrolovat opatření, uvedl Hamáček

Policisté budou s hygieniky častěji kontrolovat opatření, uvedl Hamáček
West Ham hrozí gigantům, okouzlil i Linekera. Součka je snadné přehlédnout
1. 11.  Felix
Aktuálně+

West Ham hrozí gigantům, okouzlil i Linekera. Součka je snadné přehlédnout

Aleš Vávra Aleš Vávra
před 2 hodinami
Zázrak jedné sezony? West Ham dává pochybovačům stále pádnější odpovědi a fotbalový svět si začíná uvědomovat, že se absolutní anglická fotbalová elita rozrůstá o nového člena. Tým manažera Davida Moyese prohání giganty i v aktuálním ročníku Premier League.
Spokojený Tomáš Souček po vítězství ve Villa Parku.
Spokojený Tomáš Souček po vítězství ve Villa Parku. | Foto: Reuters

Pět vítězných soutěžních duelů v řadě, během nich jediný inkasovaný gól. Čtvrté místo v lize, stejný bodový zisk jako loňský šampion Manchester City a nadšené ohlasy z tábora těch nejrenomovanějších komentátorů ostrovního fotbalu.

Související

Řepka a spol. zůstali na trávníku sami. Jejich soupeř odešel na protest ze hřiště

Fotbalisté Červených Janovic v utkání s béčkem Tupadel

West Ham je opět v kurzu, nadšené ohlasy po nedělní jasné výhře 4:1 na hřišti Aston Villy zaplnily anglický mediální prostor.

"Stali se excelentním týmem. Jsou skvělí ve všech částech hřiště a David Moyes si zaslouží obrovský kredit za to, do jaké pozice je dostal," píše na Twitter Gary Lineker.

"Nenapadá mě jediný důvod, proč by letos nemohli skončit v elitní čtyřce," přidává se Emile Heskey, někdejší útočník Liverpoolu. "Je fér říct, že vypadají fantasticky. Moyes je neskutečně oživil."

I Heskey si všiml, že se Kladiváři skvěle vyrovnávají s náročným programem a pro ně novou rolí: účastí ve více soutěžích najednou. Moyes zůstává konzervativní v určování základní sestavy, chytře ale rozšířil kádr a v Evropské lize či ligovém poháru nechává některé opory odpočívat. Výjimkou potvrzující pravidlo je přitom Tomáš Souček, o jehož nezbytnosti bude řeč níže.

"Klíčová věc je ta, že když udělá změny, pořád jim zůstává stejná struktura. To je něco, co pravidelně říkáme třeba o Manchesteru City. Ve hře neustále zůstává nějaká fundamentální filosofie. West Ham to má podobně a už kvůli tomu je třeba před Moyesem smeknout," přirovnává Heskey.

Podívejte se na důležité momenty zápasu Aston Villa - West Ham:

V Evropské lize má West Ham po třech zápasech plný bodový zisk. V anglickém ligovém poháru dobyl čtvrtfinále, když vyřadil oba bohaté velkokluby z Manchesteru.

Čeští fotbalisté nicméně momentálně nejsou ve světlech těch nejjasnějších reflektorů.

Vladimír Coufal už sice uzdravil poraněné tříslo, v sestavě ale před ním dostal přednost rozjetý Ben Johnson. Anglický mladík další působivé představení okořenil parádním gólem a potvrdil, že se stává tvrdou konkurencí pro českého reprezentačního beka.

Tomáš Souček zůstává nepostradatelným členem základní sestavy, navzdory tomu, že jeho poslední výkony působí nenápadně.

"Pořád toho odvádí strašnou spoustu mimo hlavní pozornost. Jsou to důležité věci, které je snadné přehlédnout," píše ve svém hodnocení server Claret and Hugh. 

Související

Slavný Scholes namíchl fanoušky West Hamu. Jeho slova o Součkovi berou jako urážku

Coufal, Souček, Diop (West Ham - Manchester City)

"S Declanem Ricem vytvořil silné partnerství a udělal spoustu těžké práce. Má dobrou rozehrávku. Jediné, na co si lze stěžovat, jsou jeho občasná špatná rozhodnutí ve finální třetině hřiště," hodnotí českého středopolaře londýnský večerník Evening Standard.

Web Football.London to vidí podobně. "Opět byl silný ve vzduchu, na obou koncích hřiště. Ve finální fázi se ale nerozhodoval dobře, příliš často volil špatnou variantu."

Moyes nicméně nenechává Součka oddechnout. V pěti posledních utkáních, které West Ham odehrál během pouhých čtrnácti dnů, chyběl Čech jen pár minut v závěru na Evertonu, když utrpěl zranění v obličeji.

Fanoušci pravidelně spekulují o únavě, skotský manažer ale - jak se zdá - bude mít v sestavě raději unaveného Součka než kohokoli jiného. Zvlášť, když Alex Král, plánovaný back-up do středu zálohy, stále není k dispozici.

Zatímco v minulé sezoně Souček častokrát zastínil svého kolegu Rice, letos je to právě anglický reprezentant, kdo si užívá zasloužené ódy na svou adresu.

"Hraje prostě velkolepě a připomínám, že je mu stále jen dvaadvacet let," kroutí hlavou Lineker. Není sám. Ještě před pár měsíci se většina odborníků pozastavovala nad údajnou cenovkou kolem 100 milionů liber. Nyní už zaznívají hlasy o tom, jak může být i tato hranice při případném přestupu Declana Rice výrazně překročena.

Declan Rice po vítězství na Aston Ville.
Declan Rice po vítězství na Aston Ville. | Foto: Reuters

S blížícím se zimním přestupním termínem budou spekulace nabývat na síle, fanoušci Hammers ale věří, že Rice zůstane nejméně do léta. Jeho spokojenost je do očí bijící, stejně jako ochota nechat na hřišti všechno ve prospěch Clarets and Blues.

"Náš kolektiv je teď opravdu speciální. Působíme ve výjimečném prostředí. Každé ráno se probouzíme s obrovskou touhou po dalším tréninku. Jsme nadšení," tvrdí mladá anglická superstar.

"Jsme na děleném třetím místě. Lidé se před sezonou hodně ptali, zda to můžeme dokázat znovu. Ukázali jsme, že ano. Ale musíme pokračovat. Tohle musí být náš standard. Nesmíme polevit, pokud chceme být velkým týmem," zdůrazňuje Rice.

 

Pokud jste v článku zaznamenali chybu nebo překlep, dejte nám, prosím, vědět prostřednictvím kontaktního formuláře. Děkujeme!

Související

Krmenčík divoce slavil gól "své" Plzni. On je ta největší ostuda, čílil se Limberský

Krmenčík divoce slavil gól "své" Plzni. On je ta největší ostuda, čílil se Limberský
29 fotografií

Šílenost v Argentině. Kouče postřelil při zápase fanoušek, hráči prchali z trávníku

Šílenost v Argentině. Kouče postřelil při zápase fanoušek, hráči prchali z trávníku
0:46

Agüera převezli do nemocnice. Hráč Barcelony měl problémy s dýcháním

Agüera převezli do nemocnice. Hráč Barcelony měl problémy s dýcháním

Barák pomohl skolit zvučného soupeře: Verona porazila Juventus a je osmá

Barák pomohl skolit zvučného soupeře: Verona porazila Juventus a je osmá
zahraniční ligy Fotbal sport Premier League West Ham United F.C. Tomáš Souček Declan Rice Vladimír Coufal

Právě se děje

před 1 minutou
Foto

Martin Stranka fotí, jako by točil hollywoodský film. Teď se vrací s velkou výstavou

Martin Stranka fotí, jako by točil hollywoodský film. Teď se vrací s velkou výstavou
Prohlédnout si 19 fotografií
Until You Wake Up (snímek ze souboru I Found the Silence). Autor své fotografie prezentuje pod anglickými názvy, a to jak na výstavě, tak i ve výpravném knižním katalogu. Ponecháváme je tedy v této podobě, bez překladu.
I Can Hear You Call (snímek ze souboru I Found the Silence)
před 3 minutami

Praha postaví podél Radlické ulice téměř tři stovky družstevních bytů

Podél Radlické ulice v Praze 5 vzniknou dva domy s až 266 družstevními byty. Pražští radní v pondělí schválili memorandum s pátou městskou částí, podle kterého v této lokalitě chce Praha založit bytové družstvo a byty vybudovat. Cílem je přispět k rozšíření možností cenově dostupného bydlení pro obyvatele na území Prahy. Řekla to radní Hana Kordová Marvanová (za STAN).

Projekt podpory dostupného družstevního bydlení schválilo zastupitelstvo Prahy v prosinci 2020. Město pak hledalo vhodné pozemky a z nich zatím vybralo ty na Praze 5. Nedostatek bytů a vysoká cena trápí město dlouhodobě.

"Praha chce tímto reagovat na dramatickou situaci s bydlením. Ceny bytů zde letí nahoru šestkrát rychleji než příjmy. Družstevní bydlení je tou nejrychlejší cestou, jak můžeme obyvatelům Prahy pomoci získat cenově dostupnější bydlení," řekla Marvanová.

Po uzavření memoranda se následující kroky budou týkat především ekonomických a právních aspektů spojených se založením družstva. To znamená například vytvořit detailní záměr projektu, zajistit potřebnou dokumentaci a zpracovat odhad nákladů na stavbu.

Dům na Radlické by měl mít po dokončení v příštích letech hrubou podlažní plochu zhruba 20 000 metrů čtverečních a skládat se bude minimálně ze dvou bloků se samostatnými vchody. Uvnitř bude 221 až 266 bytů a jejich průměrná výměra se má pohybovat od 54 do 65 metrů čtverečních.

Zdroj: ČTK
Aktualizováno před 21 minutami
Starostové vyzývali ke kroužkování, porušili koaliční dohodu, tvrdí analýza Pirátů
Domácí

Starostové vyzývali ke kroužkování, porušili koaliční dohodu, tvrdí analýza Pirátů

Záležitost podle předsedy Pirátů Ivana Bartoše nejprve proberou stranické orgány, jejich závěry pak projednají Piráti se STAN.
před 42 minutami
Švancara: Mosquera mě zvedl ze židle, Krmenčík nesmí na padesát kilometrů k Plzni
Česká liga

Švancara: Mosquera mě zvedl ze židle, Krmenčík nesmí na padesát kilometrů k Plzni

„Duel obhájce titulu s lídrem soutěže, tedy Slavie s Plzní, nebyl žádný fotbalový šlágr, víc mě bavil zápas Ostravy se Spartou,“ říká Petr Švancara.
Aktualizováno před 1 hodinou
"Řiďte se pravidly, ať se zase nevidíme on-line." Řada podniků nová opatření uvítala
Ekonomika

"Řiďte se pravidly, ať se zase nevidíme on-line." Řada podniků nová opatření uvítala

Řada podniků vyzývá zákazníky, aby respektovali nové nařízení, podle kterého při vstupu do restaurace musí prokázat bezinfekčnost.
Aktualizováno před 1 hodinou
Policisté budou s hygieniky častěji kontrolovat opatření, uvedl Hamáček
Domácí

ŽIVĚ Policisté budou s hygieniky častěji kontrolovat opatření, uvedl Hamáček

Události kolem pandemie covidu-19 v Česku i ve světě sledujeme v on-line reportáži.
Martin Fendrych
Martin Fendrych

Komentář: Trend se mění. Lháře, opilce a podvodníka už lidé na Hrad možná nechtějí

Další zprávy
Zprávy
Finance
Názory
Magazín
Sport
Žena
Tiráž  |  Napište nám  |  RSS  |  Hlavní zprávy do e-mailu

Centrum.cz  |  Atlas.cz 1999 – 2021 © Economia, a.s.  |  O nás  |  Všechny služby  |  Volná místa

Inzerce  |  Služby firmám  |  Všeobecné podmínky  |  Cookies  |  Nastavení soukromí  |  Ochrana osobních údajů  |  Zpracování osobních údajů  |  Audiovizuální mediální služby  |  Nápověda

Jakékoliv užití obsahu, včetně převzetí článků je bez souhlasu Economia, a.s. zapovězeno.

================================================ FILE: test/test-pages/archive-of-our-own/expected-metadata.json ================================================ { "title": "Conversations with a Cryptid - Chapter 1 - AMournfulHowlInTheNight - 僕のヒーローアカデミア | Boku no Hero Academia", "byline": "Organization for Transformative Works", "dir": null, "lang": "en", "excerpt": "An Archive of Our Own, a project of the Organization for Transformative Works", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/archive-of-our-own/expected.html ================================================

Chapter Text

Izuku was struggling to understand how he had even managed to get here, seated before the archvillain of Japan with only a sense of dread to keep him company. All Might sat concealed in an observation room, of the firm opinion that he could only aggravate the prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s memory. It started with a simple conversation gone astray on a long drive home.

“So, who is All For One? Do we know anything about him beyond what you told me before? He’s been imprisoned for months now.” Izuku remembered asking All Might from the backseat of the car as Detective Tsukauchi leisurely drove along a sprawling highway.

Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo. Izuku caught the phrase “liquid body” from the female reporter before Detective Tsukauchi changed the channel.

“Nope. Still nothing. No one really wants to speak to him,” All Might had replied brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly obstructed shoulders in the front seat shrugged. “Not much you can do with someone like him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand in a circular motion towards the side of his head.

“No one’s even made it through a full interview with him, from what I’ve heard,” Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The prison also has a “no recent events” policy on any discussions with him as well. Just in case he ends up with ideas or has some means of communicating. Given that people only want to ask him about current events, it doesn’t leave much to talk about.”

“Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t if there’s still an information block on visits.”

“Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone apart from him even knows how his Quirk works. They could get matches with any number of people, but if they’re not in a database then we can’t cross-reference them anyway. Even if they run an analysis, the data doesn’t mean anything without the ability to interpret it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial tests were conducted. They weren’t game to MRI him either, given he’s definitely got a Quirk that creates metal components.”

“No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He must be around two-hundred years old and people can’t think of a single non-current affairs thing to ask him?”

In some ways it was unfathomable that they’d let a potential resource go to waste. On the other hand, said potential resource had blown up a city, murdered numerous people and terrorised Japan for over a century. At the very least.

“Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really. Some garbage about you being too dependent on me and him letting Shigaraki run wild and how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too much time talking about nothing.”

Izuku shifted his head onto his arm. “But, that’s not really nothing, is it?”

“What do you mean?” Izuku had the feeling that All Might would have been looking at him with the you’re about to do something stupid aren’t you expression that was thankfully becoming less common.

“Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re just going to let go of me after not even two years of being taught. Maybe Shigaraki was dependent on adult figures, but I don’t even remember my dad and mum’s been busy working and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on the disobedient side of the scale.” All Might outright giggled.

“I’ll say, especially after what happened with Overhaul. I’m surprised your mother let you leave the dorms again after that.”

“I’m surprised she didn’t withdraw and ground me until I was thirty.”

“Oh? That strict?” Tsukauchi asked.

“She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an itch at the back of his head, a feeling of something missing that poked and prodded at his senses.

All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid it!”

“That’s exactly why I’m asking her and not you,” Izuku grinned from the backseat.

“He’s evil!”

“He’s ancient. You honestly don’t wonder about the sort of things someone with that life experience and Quirk would have run across to end up the way he did?”

“Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All Might snipped through folded arms.

“Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does nothing but lie, then that’s probably one too, but there’s a grain of truth in there somewhere.”

“What would you even do? Harass him into telling you his life story?” All Might sighed.

“Not when I can kill him with kindness. Who knows, it might even be poisonous for him.”

“You’re explaining this to your mother. Teacher or not, I’m not being on the receiving end of this one.”

Izuku blinked for a moment. “You’ll let me?”

“I’m not entirely for it, but any prospective information on what influenced Shigaraki can only be a good thing. If anything goes south we can pull you out pretty easily. Just be aware of who and what you’re dealing with.” Struggling, All Might turned a serious look to Izuku around the side of the seat. “Only if your mother gives the okay.”

The conversation turned to school for the rest of the way.

It might have been curiosity or it might have been the nagging sensation that chewed at his brain for the three weeks that he researched the subject of the conversation. All For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network that was two-hundred years old. There were whispers of a shadowy figure who once ruled Japan, intermingled with a string of conspiracies and fragmented events.

Izuku had even braved the dark web, poking and prodding at some of the seedier elements of the world wide web. The internet had rumours, but the dark web had stories.

An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his sanity without any reason. His grandfather had been institutionalised, crying and repeating “he took it, he took it” until his dying days. No one could console him.

Another user spoke of a nursing home where a room full of dementia residents inexplicably became docile and no longer used their Quirks on the increasingly disturbed staff. The nursing home erupted into flames just before a court case against them commenced.

A user with neon pink text spoke of how their great-great-great-great grandmother with a longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a mask of terror. No cause had ever been found.

A hacker provided a grainy CCTV recording of a heist and a scanned collection of documents from over a century ago, where there was a flash of light and entire bank vault had been emptied. What separated it from the usual robbery was that it contained a list containing confidential information on the Quirks of the First Generation. Izuku had greedily snavelled up and saved the video and documents to an external hard drive.

Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the uncle had simply disappeared.

Numerous and terrifying, the stories were scattered nuggets of gold hidden across the web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots.

After thirty-two identity changes (all carefully logged in a separate notebook), a large amount of feigning communal interest in a lucky tabloid article on All For One which had been released at the start of the first of the three weeks, Izuku hung up his tinfoil hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf and lodged his hard drive in a gap containing seven others and went to dinner.

It took another week to present his research to All Might and Tsukauchi, whose jaws reached the proverbial floor.

“We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all of it?”

“I asked the right people. Turns out criminals have very long and very unforgiving memories,” Izuku explained through sunken eyes. “There’s more than this that could be linked to him, but these ones seem to be the most obvious.”

“They would do, you can’t be head of the underworld without making an army of enemies,” All Might agreed. “You know, if you can get any more information about these events, I think you’ll give people a lot of peace of mind.”

“Provided mum agrees to it.”

“Only if she agrees to it.”

It took another month to convince his mother, who eventually gave in once All Might provided an extremely comprehensive schedule of how the visitations and any resulting research would be carefully balanced against Izuku’s schoolwork and internship.

The day of the visit finally arrived, four months after the initial conversation, much to Izuku’s dismay.

Izuku remembered how he had arrived, with the Detective and All Might escorting him through its sterile, white innards. A list of rules rattled off at the gate, “no current affairs” was chief among them and an assertion that he’d be dragged from the room if need be if Izuku was to breach any of them. No smuggling of communication devices, no weapons, no Quirks, nothing that could compromise the prisoner’s secure status.

Heavily armoured and drilled guards leading him underground into the deepest bowels of the Tartarus complex.

Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed across it in haphazard English and clutching at a carefully screened and utterly blank notebook.

Across from him, behind reinforced glass, the archvillain of Japan was bound and unmoving.

“Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed the threshold, a memory of the encounter and escape at the Kamino Ward months ago.

“Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself? Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had already lunged for the figurative jugular.

A stray thought of how does he know who I am if he’s blind and isn’t familiar with me? whispered its way through Izuku’s head.

“Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked if I could talk to you. This isn’t exactly hero related.”

“I’m surprised he said yes.” While there was little by way of expression, Izuku could just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a duck’s back.

“Well, he’s not my legal guardian, so I think you should be more surprised that mum said yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave the okay, but that was a stressful discussion.” And there it was, a miniscule twitch from the man opposite. A spasm more than anything else. Interesting. Pinned down as he was, the prisoner oozed irritation.

“At least your mother is a wise person. I wonder why the student doesn’t heed all of the advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan would have hated the comparison, but the hostility had an almost comfortable familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but here you are.”

Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out about the mental comparison as he replied. “I don’t think it’s normal for anyone my age to listen completely to their teachers. We pick and choose and run with what works best for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared more about the time management aspect of it."

“Is that a recent development?” All For One probed.

“Not really. My old homeroom teacher told me not to bother applying to U.A.” His mother’s beaming face had carried Izuku through the cheerful and resolute signing of that application form.

“I see you followed their advice to the letter,” came the snide, dismissive reply.

Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as he did so as to better prop up his notebook.

“You’re a walking contrarian, aren’t you? All Might told me about his run ins with you. What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to rationalise it as a negative and go on the attack anyway. What you’re currently doing is drawing attention away from yourself and focusing it on me so you can withhold information.” Izuku flipped open his notebook and put pen to paper. “You’ve got something fairly big to hide and you diverting attention exposes that motivation as existing anyway. The only real questions here are what and why?” Izuku paused in mortification as the man opposite’s lips parted. “I just said that aloud, didn’t I?”

Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have expected a violent outburst. In this situation, he would have expected another scathing comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he spoke, “I’ll have to change tactics, if that one’s too transparent for you. How refreshing.”

Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku carried on. “I add it to the list when you do. I’m not emotionally involved enough to really be impacted by what you’re saying. I know about you in theory, but that’s it. Maybe All Might has a history with you, but I don’t really know enough about you personally to…”

“Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only here to satisfy your curiosity as to whether or not the stories were true.”

Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually here to ask you about how your Quirk works.” For now.

Another chortle, more restrained that the last.

"What makes you think others haven’t already asked?” Had All For One been unrestrained, Izuku could imagine the stereotypical scene of the villain confidently leaning back in some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was any indication. Deflections aside, the man easily rose to each comment.

“Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers didn’t matter if the people involved were too attached to read into the answers. If none of the interviewers had managed a full interview, then it seemed unlikely that any sort of effort was put into understanding the villain.

“And you think you can? What expertise do you hold above theirs?” Doubt and reprimand weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely evil one.

Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad categories of function. You can take and give, consent doesn’t seem to be a factor. You either can’t “see” certain types of Quirks or you need to have prior knowledge of it before you take it with what I know about your brother. Despite your nom de guerre, because we both know it’s not your real name, you have a history of giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t impacted by those same restrictions, so it must either alter your brain mapping or adjust functions to allow for simultaneous use and storage. It also must isolate or categories the Quirks you stock, because from the few people who do remember you, you creating certain Quirks is always in the context of giving them to someone else meaning there’s probably an inherent immunity to stop it from tainting your own Quirk with a mutation,” Izuku mumbled, almost to himself. “The only thing really in question about your Quirk is the finer details and whether or not you need to maintain those features or if they’re inherent and your hard limit for holding Quirks.”

There was silence, for only a moment. “If only my hands were free, I would clap for such a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred. “Speculate away.” A wide and slightly unhinged smile was directed at Izuku.

It was all Izuku could do not to wince at the eagerness. An image of a nervous All Might, hidden in the observation room above with the grim-faced prison staff, came to mind.

“I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily jotted it down in his notebook. “You don’t seem bothered by the guess.”

“Few people live long enough to question my Quirk, let alone have the talent to guess so thoughtfully at its functions. It seems we share a hobby.” There was something terribly keen in that voice that hadn’t been there before, twisting itself through the compliment.

“I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged, unease uncoiling itself from the back of his mind.

“I was playing along out of boredom,” All For One corrected smoothly. “Now, I’m curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too hasty in my assessments before.”

“I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on the conversation as the words on his page began to drastically expand to distract himself from the building anxiety.

“Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim yourself has often left me wondering…”

“You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for you or if you’re doing it because I’m close to being right about your Quirk. That being said, I don’t think you know what a normal teenager is if Shigaraki is any indication. He’s about seven years too late for his rebellious phase.”

“I’m hurt and offended,” came the amused reply.

“By how Shigaraki ended up or your parenting? You only have yourself to blame for both of them.”

“How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for all of the hard work,” All For One laid out invitingly. Perhaps someone else would have risen to the bait, but Izuku was already packing his mental bags and heading for the door.

Clearly the prisoner’s anticipation had registered poorly with someone in the observation room, because a voice rang through the air. “Time’s up Midoriya-kun.”

“Okay!” Izuku called back and etched out his last thoughtful of words, untangled his legs and rose to his feet.

“What a shame, my visitations are always so short,” All For One spoke mournfully.

“Well, you did blow up half a city. They could have just let you suffocate instead. Same time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was followed out the door by wheezing laughter.

It was only after he had made it safely back to the communal room where All Might waited did he allow the spring to fade from his step and discard his nervous smile. Shuddering, he turned to All Might whose face was set in a grimace.

“I won’t say I told you so,” All Might offered, perched on the edge of his couch like a misshapen vulture.

“He’s… not really what I was expecting. I was expecting someone, more openly evil.” Izuku allowed himself to collapse into the leather of the seat. He shakily reached for the warm tea that had been clearly been prepared the moment Izuku left the cell. “I suppose he does it to lull people into a false sense of security. I didn’t understand how someone with only half a set of expressions could have “villain” written all over them until I met him.”

“He’s always been like that. He feigns concern and sympathy to lure in society’s outcasts. They’re easy targets,” All Might said through a mouthful of biscuit.

“Has he ever tried it on any of the One For All successors?”

“Not really, but you might have accidentally given him the incentive for it. He never had access to any of the One For All wielders while they were young.” All Might snorted, “not that it’ll make a difference with you”.

“I think he was trying to gauge me for a world view before the wardens ended it. I need more time to work out his response to the stuff on his Quirk.”

“He’s conversation starved since it’s solitary confinement. If what the people monitoring his brain activity said was true, you’re the most exciting thing to have happened to him in months. He replied after you left, said he was looking forward to it.”

“That’s pretty sad."

“It’s even sadder that we’re the only two members of the public who have had anything to do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,” All Might waved a tea spoon. “That’s what he gets.”

“Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his tea and headed for the exit, with him and All Might reaching it at roughly the same amount of time.

“At least your mum’s making katsudon for us tonight," was All Might's only optimistic comment.

Anxiety was still ebbing over Izuku after Tsukauchi had been debriefed in the car.

“It seems we share a hobby.” Haunted Izuku on the drive home. As if ripping someone’s Quirk from them and leaving them lying traumatised on the ground was just a fun pastime and not an act of grievous bodily harm.

And he’d be dealing with him again in another week.

================================================ FILE: test/test-pages/archive-of-our-own/source.html ================================================ Conversations with a Cryptid - Chapter 1 - AMournfulHowlInTheNight - 僕のヒーローアカデミア | Boku no Hero Academia | My Hero Academia [Archive of Our Own]

 

Actions

Work Header

Conversations with a Cryptid

Chapter Text

Izuku was struggling to understand how he had even managed to get here, seated before the archvillain of Japan with only a sense of dread to keep him company. All Might sat concealed in an observation room, of the firm opinion that he could only aggravate the prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s memory. It started with a simple conversation gone astray on a long drive home.

 

“So, who is All For One? Do we know anything about him beyond what you told me before? He’s been imprisoned for months now.” Izuku remembered asking All Might from the backseat of the car as Detective Tsukauchi leisurely drove along a sprawling highway.

Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo. Izuku caught the phrase “liquid body” from the female reporter before Detective Tsukauchi changed the channel.

“Nope. Still nothing. No one really wants to speak to him,” All Might had replied brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly obstructed shoulders in the front seat shrugged. “Not much you can do with someone like him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand in a circular motion towards the side of his head.

“No one’s even made it through a full interview with him, from what I’ve heard,” Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The prison also has a “no recent events” policy on any discussions with him as well. Just in case he ends up with ideas or has some means of communicating. Given that people only want to ask him about current events, it doesn’t leave much to talk about.”

“Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t if there’s still an information block on visits.”

“Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone apart from him even knows how his Quirk works. They could get matches with any number of people, but if they’re not in a database then we can’t cross-reference them anyway. Even if they run an analysis, the data doesn’t mean anything without the ability to interpret it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial tests were conducted. They weren’t game to MRI him either, given he’s definitely got a Quirk that creates metal components.”

“No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He must be around two-hundred years old and people can’t think of a single non-current affairs thing to ask him?”

In some ways it was unfathomable that they’d let a potential resource go to waste. On the other hand, said potential resource had blown up a city, murdered numerous people and terrorised Japan for over a century. At the very least.

“Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really. Some garbage about you being too dependent on me and him letting Shigaraki run wild and how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too much time talking about nothing.”

Izuku shifted his head onto his arm. “But, that’s not really nothing, is it?”

“What do you mean?” Izuku had the feeling that All Might would have been looking at him with the you’re about to do something stupid aren’t you expression that was thankfully becoming less common.

“Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re just going to let go of me after not even two years of being taught. Maybe Shigaraki was dependent on adult figures, but I don’t even remember my dad and mum’s been busy working and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on the disobedient side of the scale.” All Might outright giggled.

“I’ll say, especially after what happened with Overhaul. I’m surprised your mother let you leave the dorms again after that.”

“I’m surprised she didn’t withdraw and ground me until I was thirty.”

“Oh? That strict?” Tsukauchi asked.

“She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an itch at the back of his head, a feeling of something missing that poked and prodded at his senses.

All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid it!”

“That’s exactly why I’m asking her and not you,” Izuku grinned from the backseat.

“He’s evil!”

“He’s ancient. You honestly don’t wonder about the sort of things someone with that life experience and Quirk would have run across to end up the way he did?”

“Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All Might snipped through folded arms.

“Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does nothing but lie, then that’s probably one too, but there’s a grain of truth in there somewhere.”

“What would you even do? Harass him into telling you his life story?” All Might sighed.

“Not when I can kill him with kindness. Who knows, it might even be poisonous for him.”

“You’re explaining this to your mother. Teacher or not, I’m not being on the receiving end of this one.”

Izuku blinked for a moment. “You’ll let me?”

“I’m not entirely for it, but any prospective information on what influenced Shigaraki can only be a good thing. If anything goes south we can pull you out pretty easily. Just be aware of who and what you’re dealing with.” Struggling, All Might turned a serious look to Izuku around the side of the seat. “Only if your mother gives the okay.”

The conversation turned to school for the rest of the way.

 

It might have been curiosity or it might have been the nagging sensation that chewed at his brain for the three weeks that he researched the subject of the conversation. All For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network that was two-hundred years old. There were whispers of a shadowy figure who once ruled Japan, intermingled with a string of conspiracies and fragmented events.

Izuku had even braved the dark web, poking and prodding at some of the seedier elements of the world wide web. The internet had rumours, but the dark web had stories.

An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his sanity without any reason. His grandfather had been institutionalised, crying and repeating “he took it, he took it” until his dying days. No one could console him.

Another user spoke of a nursing home where a room full of dementia residents inexplicably became docile and no longer used their Quirks on the increasingly disturbed staff. The nursing home erupted into flames just before a court case against them commenced.

A user with neon pink text spoke of how their great-great-great-great grandmother with a longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a mask of terror. No cause had ever been found.

A hacker provided a grainy CCTV recording of a heist and a scanned collection of documents from over a century ago, where there was a flash of light and entire bank vault had been emptied. What separated it from the usual robbery was that it contained a list containing confidential information on the Quirks of the First Generation. Izuku had greedily snavelled up and saved the video and documents to an external hard drive.

Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the uncle had simply disappeared.

Numerous and terrifying, the stories were scattered nuggets of gold hidden across the web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots.

After thirty-two identity changes (all carefully logged in a separate notebook), a large amount of feigning communal interest in a lucky tabloid article on All For One which had been released at the start of the first of the three weeks, Izuku hung up his tinfoil hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf and lodged his hard drive in a gap containing seven others and went to dinner.

It took another week to present his research to All Might and Tsukauchi, whose jaws reached the proverbial floor.

“We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all of it?”

“I asked the right people. Turns out criminals have very long and very unforgiving memories,” Izuku explained through sunken eyes. “There’s more than this that could be linked to him, but these ones seem to be the most obvious.”

“They would do, you can’t be head of the underworld without making an army of enemies,” All Might agreed. “You know, if you can get any more information about these events, I think you’ll give people a lot of peace of mind.”

“Provided mum agrees to it.”

“Only if she agrees to it.”

It took another month to convince his mother, who eventually gave in once All Might provided an extremely comprehensive schedule of how the visitations and any resulting research would be carefully balanced against Izuku’s schoolwork and internship.

 

The day of the visit finally arrived, four months after the initial conversation, much to Izuku’s dismay.

Izuku remembered how he had arrived, with the Detective and All Might escorting him through its sterile, white innards. A list of rules rattled off at the gate, “no current affairs” was chief among them and an assertion that he’d be dragged from the room if need be if Izuku was to breach any of them. No smuggling of communication devices, no weapons, no Quirks, nothing that could compromise the prisoner’s secure status.

Heavily armoured and drilled guards leading him underground into the deepest bowels of the Tartarus complex.

Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed across it in haphazard English and clutching at a carefully screened and utterly blank notebook.

Across from him, behind reinforced glass, the archvillain of Japan was bound and unmoving.

“Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed the threshold, a memory of the encounter and escape at the Kamino Ward months ago.

“Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself? Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had already lunged for the figurative jugular.

A stray thought of how does he know who I am if he’s blind and isn’t familiar with me? whispered its way through Izuku’s head.

“Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked if I could talk to you. This isn’t exactly hero related.”

“I’m surprised he said yes.” While there was little by way of expression, Izuku could just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a duck’s back.

“Well, he’s not my legal guardian, so I think you should be more surprised that mum said yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave the okay, but that was a stressful discussion.” And there it was, a miniscule twitch from the man opposite. A spasm more than anything else. Interesting. Pinned down as he was, the prisoner oozed irritation.

“At least your mother is a wise person. I wonder why the student doesn’t heed all of the advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan would have hated the comparison, but the hostility had an almost comfortable familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but here you are.”

Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out about the mental comparison as he replied. “I don’t think it’s normal for anyone my age to listen completely to their teachers. We pick and choose and run with what works best for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared more about the time management aspect of it."

“Is that a recent development?” All For One probed.

“Not really. My old homeroom teacher told me not to bother applying to U.A.” His mother’s beaming face had carried Izuku through the cheerful and resolute signing of that application form.

“I see you followed their advice to the letter,” came the snide, dismissive reply.

Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as he did so as to better prop up his notebook.

“You’re a walking contrarian, aren’t you? All Might told me about his run ins with you. What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to rationalise it as a negative and go on the attack anyway. What you’re currently doing is drawing attention away from yourself and focusing it on me so you can withhold information.” Izuku flipped open his notebook and put pen to paper. “You’ve got something fairly big to hide and you diverting attention exposes that motivation as existing anyway. The only real questions here are what and why?” Izuku paused in mortification as the man opposite’s lips parted. “I just said that aloud, didn’t I?”

Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have expected a violent outburst. In this situation, he would have expected another scathing comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he spoke, “I’ll have to change tactics, if that one’s too transparent for you. How refreshing.”

Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku carried on. “I add it to the list when you do. I’m not emotionally involved enough to really be impacted by what you’re saying. I know about you in theory, but that’s it. Maybe All Might has a history with you, but I don’t really know enough about you personally to…”

“Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only here to satisfy your curiosity as to whether or not the stories were true.”

Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually here to ask you about how your Quirk works.” For now.

Another chortle, more restrained that the last.

"What makes you think others haven’t already asked?” Had All For One been unrestrained, Izuku could imagine the stereotypical scene of the villain confidently leaning back in some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was any indication. Deflections aside, the man easily rose to each comment.

“Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers didn’t matter if the people involved were too attached to read into the answers. If none of the interviewers had managed a full interview, then it seemed unlikely that any sort of effort was put into understanding the villain.

“And you think you can? What expertise do you hold above theirs?” Doubt and reprimand weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely evil one.

Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad categories of function. You can take and give, consent doesn’t seem to be a factor. You either can’t “see” certain types of Quirks or you need to have prior knowledge of it before you take it with what I know about your brother. Despite your nom de guerre, because we both know it’s not your real name, you have a history of giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t impacted by those same restrictions, so it must either alter your brain mapping or adjust functions to allow for simultaneous use and storage. It also must isolate or categories the Quirks you stock, because from the few people who do remember you, you creating certain Quirks is always in the context of giving them to someone else meaning there’s probably an inherent immunity to stop it from tainting your own Quirk with a mutation,” Izuku mumbled, almost to himself. “The only thing really in question about your Quirk is the finer details and whether or not you need to maintain those features or if they’re inherent and your hard limit for holding Quirks.”

There was silence, for only a moment. “If only my hands were free, I would clap for such a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred. “Speculate away.” A wide and slightly unhinged smile was directed at Izuku.

It was all Izuku could do not to wince at the eagerness. An image of a nervous All Might, hidden in the observation room above with the grim-faced prison staff, came to mind.

“I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily jotted it down in his notebook. “You don’t seem bothered by the guess.”

“Few people live long enough to question my Quirk, let alone have the talent to guess so thoughtfully at its functions. It seems we share a hobby.” There was something terribly keen in that voice that hadn’t been there before, twisting itself through the compliment.

“I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged, unease uncoiling itself from the back of his mind.

“I was playing along out of boredom,” All For One corrected smoothly. “Now, I’m curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too hasty in my assessments before.”

“I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on the conversation as the words on his page began to drastically expand to distract himself from the building anxiety.

“Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim yourself has often left me wondering…”

“You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for you or if you’re doing it because I’m close to being right about your Quirk. That being said, I don’t think you know what a normal teenager is if Shigaraki is any indication. He’s about seven years too late for his rebellious phase.”

“I’m hurt and offended,” came the amused reply.

“By how Shigaraki ended up or your parenting? You only have yourself to blame for both of them.”

“How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for all of the hard work,” All For One laid out invitingly. Perhaps someone else would have risen to the bait, but Izuku was already packing his mental bags and heading for the door.

Clearly the prisoner’s anticipation had registered poorly with someone in the observation room, because a voice rang through the air. “Time’s up Midoriya-kun.”

“Okay!” Izuku called back and etched out his last thoughtful of words, untangled his legs and rose to his feet.

“What a shame, my visitations are always so short,” All For One spoke mournfully.

“Well, you did blow up half a city. They could have just let you suffocate instead. Same time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was followed out the door by wheezing laughter.

It was only after he had made it safely back to the communal room where All Might waited did he allow the spring to fade from his step and discard his nervous smile. Shuddering, he turned to All Might whose face was set in a grimace.

“I won’t say I told you so,” All Might offered, perched on the edge of his couch like a misshapen vulture.

“He’s… not really what I was expecting. I was expecting someone, more openly evil.” Izuku allowed himself to collapse into the leather of the seat. He shakily reached for the warm tea that had been clearly been prepared the moment Izuku left the cell. “I suppose he does it to lull people into a false sense of security. I didn’t understand how someone with only half a set of expressions could have “villain” written all over them until I met him.”

“He’s always been like that. He feigns concern and sympathy to lure in society’s outcasts. They’re easy targets,” All Might said through a mouthful of biscuit.

“Has he ever tried it on any of the One For All successors?”

“Not really, but you might have accidentally given him the incentive for it. He never had access to any of the One For All wielders while they were young.” All Might snorted, “not that it’ll make a difference with you”.

“I think he was trying to gauge me for a world view before the wardens ended it. I need more time to work out his response to the stuff on his Quirk.”

“He’s conversation starved since it’s solitary confinement. If what the people monitoring his brain activity said was true, you’re the most exciting thing to have happened to him in months. He replied after you left, said he was looking forward to it.”

“That’s pretty sad."

“It’s even sadder that we’re the only two members of the public who have had anything to do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,” All Might waved a tea spoon. “That’s what he gets.”

“Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his tea and headed for the exit, with him and All Might reaching it at roughly the same amount of time.

“At least your mum’s making katsudon for us tonight," was All Might's only optimistic comment.

Anxiety was still ebbing over Izuku after Tsukauchi had been debriefed in the car.

“It seems we share a hobby.” Haunted Izuku on the drive home. As if ripping someone’s Quirk from them and leaving them lying traumatised on the ground was just a fun pastime and not an act of grievous bodily harm.

And he’d be dealing with him again in another week.

================================================ FILE: test/test-pages/ars-1/expected-metadata.json ================================================ { "title": "Just-released Minecraft exploit makes it easy to crash game servers", "byline": "Dan Goodin", "dir": null, "lang": "en-us", "excerpt": "Two-year-old bug exposes thousands of servers to crippling attack.", "siteName": "Ars Technica", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/ars-1/expected.html ================================================

Biz & IT —

Two-year-old bug exposes thousands of servers to crippling attack.

Just-released Minecraft exploit makes it easy to crash game servers

A flaw in the wildly popular online game Minecraft makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability.

"I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a blog post published Thursday, 21 months, he said, after privately reporting the bug to Minecraft developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it."

The bug resides in the networking internals of the Minecraft protocol. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. Minecraft items can also store arbitrary metadata in a file format known as Named Binary Tag (NBT), which allows complex data structures to be kept in hierarchical nests. Askar has released proof-of-concept attack code he said exploits the vulnerability to crash any server hosting the game. Here's how it works.

The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to craft a packet that is incredibly complex for the server to deserialize but trivial for us to generate.

In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like. 

rekt: {
    list: [
        list: [
            list: [
                list: [
                    list: [
                        list: [
                        ]
                        list: [
                        ]
                        list: [
                        ]
                        list: [
                        ]
                        ...
                    ]
                    ...
                ]
                ...
            ]
            ...
        ]
        ...
    ]
    ...
}

The root of the object, rekt, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of 10^5 * 300 = 30,000,000 lists.

And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes.

Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to 2^15 - 1. Now that the length is a varint capable of storing integers up to 2^28, our potential for attack has increased significantly.

When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load.

This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the 0x08: Block Placement Packet and 0x10: Creative Inventory Action.

The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented.

These were the fixes that I recommended to Mojang 2 years ago.

Ars is asking Mojang for comment and will update this post if company officials respond.

================================================ FILE: test/test-pages/ars-1/source.html ================================================ Just-released Minecraft exploit makes it easy to crash game servers | Ars Technica
Subscribe

Biz & IT —

Just-released Minecraft exploit makes it easy to crash game servers

Two-year-old bug exposes thousands of servers to crippling attack.

-

Just-released Minecraft exploit makes it easy to crash game servers
Kevin

A flaw in the wildly popular online game Minecraft makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability.

"I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a blog post published Thursday, 21 months, he said, after privately reporting the bug to Minecraft developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it."

The bug resides in the networking internals of the Minecraft protocol. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. Minecraft items can also store arbitrary metadata in a file format known as Named Binary Tag (NBT), which allows complex data structures to be kept in hierarchical nests. Askar has released proof-of-concept attack code he said exploits the vulnerability to crash any server hosting the game. Here's how it works.

The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to craft a packet that is incredibly complex for the server to deserialize but trivial for us to generate.

In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like. 

rekt: {
    list: [
        list: [
            list: [
                list: [
                    list: [
                        list: [
                        ]
                        list: [
                        ]
                        list: [
                        ]
                        list: [
                        ]
                        ...
                    ]
                    ...
                ]
                ...
            ]
            ...
        ]
        ...
    ]
    ...
}

The root of the object, rekt, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of 10^5 * 300 = 30,000,000 lists.

And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes.

Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to 2^15 - 1. Now that the length is a varint capable of storing integers up to 2^28, our potential for attack has increased significantly.

When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load.

This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the 0x08: Block Placement Packet and 0x10: Creative Inventory Action.

The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented.

These were the fixes that I recommended to Mojang 2 years ago.

Ars is asking Mojang for comment and will update this post if company officials respond.

Dan Goodin Dan is the Security Editor at Ars Technica, which he joined in 2012 after working for The Register, the Associated Press, Bloomberg News, and other publications.
Email dan.goodin@arstechnica.com // Twitter @dangoodin001

You must to comment.

Channel Ars Technica

================================================ FILE: test/test-pages/article-author-tag/expected-metadata.json ================================================ { "title": "The Deck of Cards That Made Tarot A Global Phenomenon", "byline": "Laura June Topolsky", "dir": null, "lang": "en", "excerpt": "Picture a deck of tarot cards. What do you see? Maybe the Magician in his rich red robes, right arm raised high above him. Or the skeleton on horseback for...", "siteName": "Atlas Obscura", "publishedTime": "2015-07-10T09:53:00-04:00", "readerable": true } ================================================ FILE: test/test-pages/article-author-tag/expected.html ================================================

(All photos: Atlas staff) 

Picture a deck of tarot cards. What do you see? Maybe the Magician in his rich red robes, right arm raised high above him. Or the skeleton on horseback for Death. Or maybe you think of The Hermit in grey, holding his lantern, walking with a staff, featured in the artwork for Led Zeppelin IV. 

The funny thing is that those images don’t hail from some ancient text but from one particular deck of cards, a relatively recent one in the life of this medieval form of entertainment: The Rider (or Rider-Waite-Smith) Tarot. That you can visualize a tarot card at all is likely due to one man, an American business man named Stuart R. Kaplan, founder and chairman of U.S. Games Systems, Inc., which has been producing and selling the Rider-Waite deck since 1970.

Well, him and the artist, Pamela Colman Smith, responsible for the 78 original drawings that make up the deck. 

Tarot itself is a fairly simple game. The instructions, with each card placed in its position having a particular purpose, are strikingly simple, easy for even a first timer. With time and practice, the cards may begin to really convey meaning, asking us to look deeper at ourselves and our motives, and nudging us in the correct direction. But it took an unholy trinity to bring this to the American public. The most famous and popular tarot deck in the world, the first ever printed in English, the Rider-Waite tarot is a product of the intuitive thoughtfulness of an American-born occult scholar in late 19th century London, a British-born creative visual genius who studied art in New York City and lived in Jamaica, and a businessman whose first book was about coal mining techniques who happened upon a set of tarot cards at a toy fair in the late 1960s. These three people, essentially, are the only reason any of us know much of anything about tarot.

Tarot cards have a long story and a short one. The long one begins in 15th century Europe, where the 78 card deck—four suits (Wands, Cups, Swords, Pentacles) of 10 “pip”cards running from Ace to ten, each with an additional four “face”cards (King, Queen, Knight, and Jack) plus 21 Trump cards known as the Major Arcana, plus the Fool—form the traditional tarot deck. Without the Major Arcana, the cards of Tarot are roughly aligned with a standard playing card deck, which has its roots in France a few hundred years earlier. By the 15th century, tarot cards were widely used for popular trick-taking games such as French Tarot (which is still widely played) and the Italian game tarochini.

The shorter story, however, is how Tarot came to be used in fortune telling. In 1781, when Antoine Court de Gébelin, a French Freemason and Protestant pastor published a book called Le Monde Primitif, tracing the mysticisms of the ancient world and their surviving traces in the modern. Among them, he included the famous French playing card deck, the Tarot de Marseilles, which he connected to the Egyptian deities Isis and Thoth. Though his musings on the subject haven’t been found to be based on any evidence, tarot’s association with the mystical was now set. A few other, mostly French, writers and occultists followed in writing treatises and books on the occult leanings and possibilities offered by the cards, but tarot readings were hardly mainstream.

In 19th century England, however, interest verging on a mania raged for all things occult. Suddenly, worlds of knowledge, coupled with current thinking on the psychology of the human mind opened up, and people of all walks of life became enamored with contacting the spirit world to find out the future or to commune with the dead. Christians began reading the Kabbalah. Interest in photographing ghosts rose.

The Hermetic Order of the Golden Dawn, devoted to study of the occult, and one of the first organizations of its type to fully admit women in addition to men, was founded with its first temple in London in 1888. Called the Isis-Urania Temple, it was founded by three Freemasons who were also members of the Rosicrucian Society of England, an esoteric Christian order. Early members of the Golden Dawn included William Butler Yeats, Aleister Crowley, and Arthur Edward Waite. Several splinter temples formed over disputes—largely seeming to do with the later famous occultist Aleister Crowley, who was infamous during his lifetime for his experimentation with drugs, his libertine lifestyle, and his outspoken minority opinion for the time that homosexual desires should never be repressed or ignored. One of the reformed orders included who would become the founding team of the modern English language tarot: Arthur Edward Waite and Pamela Colman Smith.

Arthur Edward Waite was born in Brooklyn, New York in 1857 to an American father and a British mother. Arthur’s father died before he was two, leaving his mother a widow. She returned with her family to England, where Arthur spent the rest of his life. He apparently became interested in the occult when his sister died at a young age in 1874. Eventually, he joined up with the Order of the Golden Dawn, later also becoming a Freemason and then a Rosicrucian. As far as occult topics were concerned, Waite’s interests were varied and far-reaching. He wrote books on topics such as the Kabbalah, mysticism, ceremonial magic, and the Holy Grail, beginning in the late 1880s. His work was well received in mystic and academic circles, and he was soon one of the best known authorities on such topics. One of his books, Book of Black Magic and Pacts (1898) led a very young Aleister Crowley to write to Waite for advice. Eventually, through their associations with the various societies and brotherhoods, the two sparred and became “enemies,”with Crowley attacking Waite in his writings for years. There is a hint of theatrics to all of this, and the contemplative Waite seems not to have taken the bait for the most part. Certainly it didn’t slow down the prolific output of either.

Which brings us back to the tarot cards. In 1908 the British Museum acquired black and white photographs of a full deck of tarot cards now known as the Sola-Busca deck. The deck, which dates from around 1490, was extremely special, and Arthur Edward Waite would have known that when he went to see the photographs on exhibition at the museum soon after their arrival.

It is, first and foremost, the earliest extant complete tarot deck. It is also, significantly, the first deck to illustrate all of the “pip”cards: previous decks had all had a 2 of Swords, but instead of a full illustration, there would simply be two swords on the card. This deck illustrated all cards equally, with fully realized illustrations, setting its suits apart from regular old playing cards, and thereby also obscuring their relationship to the layman’s eyes. The Sola-Busca deck would prove to be inspirational to Pamela Colman Smith, the artist Waite chose to draw his deck, and in fact several of the cards in both decks are almost identical in design.

Smith’s story begins in London, where she was born to an American father and a Jamaican mother. She travelled between Jamaica, London, and New York as a child, and eventually studied art at the Pratt Institute in New York City under Arthur Wesley Dowell, though she didn’t earn a degree. She set up shop as a commercial illustrator in London and did the art for a volume of William Butler Yeats verse, several magazines, and eventually illustrated Bram Stoker’s last published work, The Lair of the White Worm, in 1911. Smith’s art was colorful and unique, and in 1907 Alfred Stieglitz, the photographer (and eventual husband of Georgia O’Keefe) gave Smith her own show at his Photo-Secession Gallery, in his first non-photographic exhibit. Like Waite, Smith’s interests were varied, and she published her own books on Jamaican folklore, as well as (briefly) her own magazine, The Green Sheaf, each issue of which bore the words, “my sheaf is small, but it is green.” Smith was known in her own time to have a “second sight,”and to paint pictures based on visions she saw while listening to music. She was ushered into the Hermetic Order of the Golden Dawn by Yeats, which she joined in 1901. She splintered off with Waite when he left the Order to form his own incarnation.

It was, then, through their association with one another that Waite came to commission Smith to create 78 original drawings for the new tarot deck, the first one in English, which he wished to create. Unlike most previous decks, Waite’s tarot would be primarily for divination and so the images would be intentionally laden with meaning. In six months, Smith completed the work, seemingly from written instructions by Waite for the Major Arcana, letting her imagination fully guide the rest of the deck. The Rider-Waite deck, as it came to be known, was published in 1909 by Rider Company in England. The next year, Waite published a small guide to reading the cards for Rider, and in 1911 he published his full book on the subject, The Pictorial Key to the Tarot. 

Smith’s drawings, under the guidance of Waite—who proposed a thoughtful way for everyday people to read the cards years before the industry of professional tarot readers sprang up—are vibrant and intuitive, and though Waite’s aim was to produce a beautiful, art-minded deck (success!) what happened was that the two created a deck capable of passing, eventually, into the mainstream consciousness.

Each of Smith’s drawings conjures enough of Waite’s key phrases—the divinatory meanings of the cards—that the pure novice is likely to guess at them on a cursory peek at the image itself. Take the Nine of Swords, for example. The image, on a stark black background, nine swords in parallel behind a figure, sitting up from the covers in bed, head in hands. The image is desolate: something awakens him or her (and so many of Smith’s figures are androgynous) in the dead of night. Is it worry or fear, or is it both? One looks at the card and sympathizes: we have all had such sleepless nights of pondering, either over the past which cannot be changed, or the present, which is confusing. You can see this in the card without ever glimpsing Waite’s accompanying text.

But here is his text:

“Divinatory Meanings: Death, failure, miscarriage, delay, deception, disappointment, despair. Reversed (when the card appears upside down in a reading): Imprisonment, suspicion, doubt, reasonable fear, shame.”

A complete newcomer to the tarot, if asked to describe the emotions the card generates might not come up with these exact words, but the meaning is certainly apparent, and obvious. And so goes each and every card, even the ones which describe more abstract thoughts and feelings. It was this mind-melding of Waite and Smith which produced the deck of tarot cards which most of us now know as “the tarot.”

When Arthur Waite died in 1942, his obituary, published in the Brooklyn Daily Eagle, called him the author of “many books on occult phenomenon.”There was no mention of his tarot deck. When Pamela Colman Smith died 16 years later… well, I can find no obituary for Smith. Tarot cards, never mainstream, didn’t exactly disappear, but they also did not flourish by any means.

Enter Stuart R. Kaplan, a graduate of the Wharton School of Business in 1955. “I’m all yours,”he tells me after a few weeks of missed connections on the phone, before launching into his personal history. Kaplan was working in New York City in the late 1960s, managing mines in West Virginia and Pennsylvania when he went to Germany on a business trip in 1968, ending up, out of curiosity, on his free time, at the Nuremberg Toy Fair. He wasn’t completely unknown in the world of games: he’d created his own Student Survival board game in the U.S., and it had seen moderate success. Still, when he happened upon the exhibition booth of the Swiss AG Müller & Cie, he found an odd deck of cards, and he was intrigued.

Kaplan didn’t know much of anything about tarot at that point, but cut a deal to import a few thousand of the decks—known as the Swiss 1JJ tarot—to sell in the United States. Kaplan targeted large bookstores such as Brentano’s, and was successful enough in his efforts that he began looking for other tarot decks to import. In the meantime, he wrote the first of his many, many books on tarot: Tarot Cards for Fun and Fortune Telling, published in 1970. In 1971 he began to sell a Marseilles-style tarot deck, and wrote a second, more detailed book, Tarot Classic. This book delved into the history of tarot, and for the first time Kaplan wrote about Arthur Edward Waite. Soon after, Kaplan negotiated with the British company that held rights to the Rider-Waite deck, which he says wasn’t selling decks at that point. Kaplan wouldn’t import the cards. Working with Rider and the blessing of Waite’s only surviving heir, his daughter Sybil, Kaplan would own the rights to publish them in the United States.

The Rider-Waite tarot, and tarot generally, had never been widely in circulation. But Kaplan had now been, for several years, selling hundreds of thousands of tarot decks quietly from the offices of his new company, U.S. Games Systems, Inc. in Stamford, Connecticut. And when he began selling Waite’s deck, there was a slow but sure explosion. “I think the full illustrations of all the cards, including the pips, is the key to the deck’s longevity and popularity,” Kaplan says. U.S. Games, which currently has about 50 decks in print and has published hundreds over the years, prints decks for a few years and then refreshes. Only the Rider-Waite decks have never gone out of print, and demand for them, Kaplan says, is always consistent.

By the mid-1970s, tarot was on its way to becoming popular in the mainstream, and almost everyone’s tarot cards came from the same place, U.S. Games Systems, Inc. “We have prevailed because we have devoted all of our attention to tarot cards,”Kaplan says. “Companies come and go: they’ll print one deck and then be gone. We have never given up,” he goes on, and because of Kaplan and his company, tarot cards are widely available and cheap. “If I see a deck on sale on eBay,” Kaplan, also an avid collector of games, rare tarot cards, and books says, “and it’s $100 or something, that upsets me. If there is a demand for an old deck that’s out of print then we’ll just bring it back. The decks aren’t supposed to be so expensive.”

Kaplan sold parts of his own personal collection of rare tarot artifacts at Christie’s in 2006, with some individual cards selling for thousands of dollars. U.S. Games sells new decks for $20. “I think tarot is popular because each deck is an unpaged book,” Kaplan says. “Shuffle them and you’ll get a new story every time.”

 

================================================ FILE: test/test-pages/article-author-tag/source.html ================================================ The Deck of Cards That Made Tarot A Global Phenomenon - Atlas Obscura
Places near me Random place

The Deck of Cards That Made Tarot A Global Phenomenon

by Laura June Topolsky July 10, 2015

(All photos: Atlas staff) 

Picture a deck of tarot cards. What do you see? Maybe the Magician in his rich red robes, right arm raised high above him. Or the skeleton on horseback for Death. Or maybe you think of The Hermit in grey, holding his lantern, walking with a staff, featured in the artwork for Led Zeppelin IV. 

The funny thing is that those images don’t hail from some ancient text but from one particular deck of cards, a relatively recent one in the life of this medieval form of entertainment: The Rider (or Rider-Waite-Smith) Tarot. That you can visualize a tarot card at all is likely due to one man, an American business man named Stuart R. Kaplan, founder and chairman of U.S. Games Systems, Inc., which has been producing and selling the Rider-Waite deck since 1970.

Well, him and the artist, Pamela Colman Smith, responsible for the 78 original drawings that make up the deck. 

Tarot itself is a fairly simple game. The instructions, with each card placed in its position having a particular purpose, are strikingly simple, easy for even a first timer. With time and practice, the cards may begin to really convey meaning, asking us to look deeper at ourselves and our motives, and nudging us in the correct direction. But it took an unholy trinity to bring this to the American public. The most famous and popular tarot deck in the world, the first ever printed in English, the Rider-Waite tarot is a product of the intuitive thoughtfulness of an American-born occult scholar in late 19th century London, a British-born creative visual genius who studied art in New York City and lived in Jamaica, and a businessman whose first book was about coal mining techniques who happened upon a set of tarot cards at a toy fair in the late 1960s. These three people, essentially, are the only reason any of us know much of anything about tarot.

 

Tarot cards have a long story and a short one. The long one begins in 15th century Europe, where the 78 card deck—four suits (Wands, Cups, Swords, Pentacles) of 10 “pip”cards running from Ace to ten, each with an additional four “face”cards (King, Queen, Knight, and Jack) plus 21 Trump cards known as the Major Arcana, plus the Fool—form the traditional tarot deck. Without the Major Arcana, the cards of Tarot are roughly aligned with a standard playing card deck, which has its roots in France a few hundred years earlier. By the 15th century, tarot cards were widely used for popular trick-taking games such as French Tarot (which is still widely played) and the Italian game tarochini.

The shorter story, however, is how Tarot came to be used in fortune telling. In 1781, when Antoine Court de Gébelin, a French Freemason and Protestant pastor published a book called Le Monde Primitif, tracing the mysticisms of the ancient world and their surviving traces in the modern. Among them, he included the famous French playing card deck, the Tarot de Marseilles, which he connected to the Egyptian deities Isis and Thoth. Though his musings on the subject haven’t been found to be based on any evidence, tarot’s association with the mystical was now set. A few other, mostly French, writers and occultists followed in writing treatises and books on the occult leanings and possibilities offered by the cards, but tarot readings were hardly mainstream.

In 19th century England, however, interest verging on a mania raged for all things occult. Suddenly, worlds of knowledge, coupled with current thinking on the psychology of the human mind opened up, and people of all walks of life became enamored with contacting the spirit world to find out the future or to commune with the dead. Christians began reading the Kabbalah. Interest in photographing ghosts rose.

The Hermetic Order of the Golden Dawn, devoted to study of the occult, and one of the first organizations of its type to fully admit women in addition to men, was founded with its first temple in London in 1888. Called the Isis-Urania Temple, it was founded by three Freemasons who were also members of the Rosicrucian Society of England, an esoteric Christian order. Early members of the Golden Dawn included William Butler Yeats, Aleister Crowley, and Arthur Edward Waite. Several splinter temples formed over disputes—largely seeming to do with the later famous occultist Aleister Crowley, who was infamous during his lifetime for his experimentation with drugs, his libertine lifestyle, and his outspoken minority opinion for the time that homosexual desires should never be repressed or ignored. One of the reformed orders included who would become the founding team of the modern English language tarot: Arthur Edward Waite and Pamela Colman Smith.

Arthur Edward Waite was born in Brooklyn, New York in 1857 to an American father and a British mother. Arthur’s father died before he was two, leaving his mother a widow. She returned with her family to England, where Arthur spent the rest of his life. He apparently became interested in the occult when his sister died at a young age in 1874. Eventually, he joined up with the Order of the Golden Dawn, later also becoming a Freemason and then a Rosicrucian. As far as occult topics were concerned, Waite’s interests were varied and far-reaching. He wrote books on topics such as the Kabbalah, mysticism, ceremonial magic, and the Holy Grail, beginning in the late 1880s. His work was well received in mystic and academic circles, and he was soon one of the best known authorities on such topics. One of his books, Book of Black Magic and Pacts (1898) led a very young Aleister Crowley to write to Waite for advice. Eventually, through their associations with the various societies and brotherhoods, the two sparred and became “enemies,”with Crowley attacking Waite in his writings for years. There is a hint of theatrics to all of this, and the contemplative Waite seems not to have taken the bait for the most part. Certainly it didn’t slow down the prolific output of either.

Which brings us back to the tarot cards. In 1908 the British Museum acquired black and white photographs of a full deck of tarot cards now known as the Sola-Busca deck. The deck, which dates from around 1490, was extremely special, and Arthur Edward Waite would have known that when he went to see the photographs on exhibition at the museum soon after their arrival.

It is, first and foremost, the earliest extant complete tarot deck. It is also, significantly, the first deck to illustrate all of the “pip”cards: previous decks had all had a 2 of Swords, but instead of a full illustration, there would simply be two swords on the card. This deck illustrated all cards equally, with fully realized illustrations, setting its suits apart from regular old playing cards, and thereby also obscuring their relationship to the layman’s eyes. The Sola-Busca deck would prove to be inspirational to Pamela Colman Smith, the artist Waite chose to draw his deck, and in fact several of the cards in both decks are almost identical in design.

Smith’s story begins in London, where she was born to an American father and a Jamaican mother. She travelled between Jamaica, London, and New York as a child, and eventually studied art at the Pratt Institute in New York City under Arthur Wesley Dowell, though she didn’t earn a degree. She set up shop as a commercial illustrator in London and did the art for a volume of William Butler Yeats verse, several magazines, and eventually illustrated Bram Stoker’s last published work, The Lair of the White Worm, in 1911. Smith’s art was colorful and unique, and in 1907 Alfred Stieglitz, the photographer (and eventual husband of Georgia O’Keefe) gave Smith her own show at his Photo-Secession Gallery, in his first non-photographic exhibit. Like Waite, Smith’s interests were varied, and she published her own books on Jamaican folklore, as well as (briefly) her own magazine, The Green Sheaf, each issue of which bore the words, “my sheaf is small, but it is green.” Smith was known in her own time to have a “second sight,”and to paint pictures based on visions she saw while listening to music. She was ushered into the Hermetic Order of the Golden Dawn by Yeats, which she joined in 1901. She splintered off with Waite when he left the Order to form his own incarnation.

It was, then, through their association with one another that Waite came to commission Smith to create 78 original drawings for the new tarot deck, the first one in English, which he wished to create. Unlike most previous decks, Waite’s tarot would be primarily for divination and so the images would be intentionally laden with meaning. In six months, Smith completed the work, seemingly from written instructions by Waite for the Major Arcana, letting her imagination fully guide the rest of the deck. The Rider-Waite deck, as it came to be known, was published in 1909 by Rider Company in England. The next year, Waite published a small guide to reading the cards for Rider, and in 1911 he published his full book on the subject, The Pictorial Key to the Tarot. 

Smith’s drawings, under the guidance of Waite—who proposed a thoughtful way for everyday people to read the cards years before the industry of professional tarot readers sprang up—are vibrant and intuitive, and though Waite’s aim was to produce a beautiful, art-minded deck (success!) what happened was that the two created a deck capable of passing, eventually, into the mainstream consciousness.

Each of Smith’s drawings conjures enough of Waite’s key phrases—the divinatory meanings of the cards—that the pure novice is likely to guess at them on a cursory peek at the image itself. Take the Nine of Swords, for example. The image, on a stark black background, nine swords in parallel behind a figure, sitting up from the covers in bed, head in hands. The image is desolate: something awakens him or her (and so many of Smith’s figures are androgynous) in the dead of night. Is it worry or fear, or is it both? One looks at the card and sympathizes: we have all had such sleepless nights of pondering, either over the past which cannot be changed, or the present, which is confusing. You can see this in the card without ever glimpsing Waite’s accompanying text.

But here is his text:

“Divinatory Meanings: Death, failure, miscarriage, delay, deception, disappointment, despair. Reversed (when the card appears upside down in a reading): Imprisonment, suspicion, doubt, reasonable fear, shame.”

A complete newcomer to the tarot, if asked to describe the emotions the card generates might not come up with these exact words, but the meaning is certainly apparent, and obvious. And so goes each and every card, even the ones which describe more abstract thoughts and feelings. It was this mind-melding of Waite and Smith which produced the deck of tarot cards which most of us now know as “the tarot.”

When Arthur Waite died in 1942, his obituary, published in the Brooklyn Daily Eagle, called him the author of “many books on occult phenomenon.”There was no mention of his tarot deck. When Pamela Colman Smith died 16 years later… well, I can find no obituary for Smith. Tarot cards, never mainstream, didn’t exactly disappear, but they also did not flourish by any means.

Enter Stuart R. Kaplan, a graduate of the Wharton School of Business in 1955. “I’m all yours,”he tells me after a few weeks of missed connections on the phone, before launching into his personal history. Kaplan was working in New York City in the late 1960s, managing mines in West Virginia and Pennsylvania when he went to Germany on a business trip in 1968, ending up, out of curiosity, on his free time, at the Nuremberg Toy Fair. He wasn’t completely unknown in the world of games: he’d created his own Student Survival board game in the U.S., and it had seen moderate success. Still, when he happened upon the exhibition booth of the Swiss AG Müller & Cie, he found an odd deck of cards, and he was intrigued.

Kaplan didn’t know much of anything about tarot at that point, but cut a deal to import a few thousand of the decks—known as the Swiss 1JJ tarot—to sell in the United States. Kaplan targeted large bookstores such as Brentano’s, and was successful enough in his efforts that he began looking for other tarot decks to import. In the meantime, he wrote the first of his many, many books on tarot: Tarot Cards for Fun and Fortune Telling, published in 1970. In 1971 he began to sell a Marseilles-style tarot deck, and wrote a second, more detailed book, Tarot Classic. This book delved into the history of tarot, and for the first time Kaplan wrote about Arthur Edward Waite. Soon after, Kaplan negotiated with the British company that held rights to the Rider-Waite deck, which he says wasn’t selling decks at that point. Kaplan wouldn’t import the cards. Working with Rider and the blessing of Waite’s only surviving heir, his daughter Sybil, Kaplan would own the rights to publish them in the United States.

The Rider-Waite tarot, and tarot generally, had never been widely in circulation. But Kaplan had now been, for several years, selling hundreds of thousands of tarot decks quietly from the offices of his new company, U.S. Games Systems, Inc. in Stamford, Connecticut. And when he began selling Waite’s deck, there was a slow but sure explosion. “I think the full illustrations of all the cards, including the pips, is the key to the deck’s longevity and popularity,” Kaplan says. U.S. Games, which currently has about 50 decks in print and has published hundreds over the years, prints decks for a few years and then refreshes. Only the Rider-Waite decks have never gone out of print, and demand for them, Kaplan says, is always consistent.

By the mid-1970s, tarot was on its way to becoming popular in the mainstream, and almost everyone’s tarot cards came from the same place, U.S. Games Systems, Inc. “We have prevailed because we have devoted all of our attention to tarot cards,”Kaplan says. “Companies come and go: they’ll print one deck and then be gone. We have never given up,” he goes on, and because of Kaplan and his company, tarot cards are widely available and cheap. “If I see a deck on sale on eBay,” Kaplan, also an avid collector of games, rare tarot cards, and books says, “and it’s $100 or something, that upsets me. If there is a demand for an old deck that’s out of print then we’ll just bring it back. The decks aren’t supposed to be so expensive.”

Kaplan sold parts of his own personal collection of rare tarot artifacts at Christie’s in 2006, with some individual cards selling for thousands of dollars. U.S. Games sells new decks for $20. “I think tarot is popular because each deck is an unpaged book,” Kaplan says. “Shuffle them and you’ll get a new story every time.”

 

Read next

FOUND: Sharks Hanging Out in a Volcano

Want to see fewer ads? Become a Member.
From Around the Web

================================================ FILE: test/test-pages/base-url/expected-metadata.json ================================================ { "title": "Base URL test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/base-url/expected.html ================================================

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

================================================ FILE: test/test-pages/base-url/source.html ================================================ Base URL test

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
================================================ FILE: test/test-pages/base-url-base-element/expected-metadata.json ================================================ { "title": "Base URL with base test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/base-url-base-element/expected.html ================================================

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

================================================ FILE: test/test-pages/base-url-base-element/source.html ================================================ Base URL with base test

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
================================================ FILE: test/test-pages/base-url-base-element-relative/expected-metadata.json ================================================ { "title": "Base URL with base relative test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/base-url-base-element-relative/expected.html ================================================

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

================================================ FILE: test/test-pages/base-url-base-element-relative/source.html ================================================ Base URL with base relative test

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Links

link

link

link

link

link

link

link

link

Images

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
================================================ FILE: test/test-pages/basic-tags-cleaning/expected-metadata.json ================================================ { "title": "Basic tag cleaning test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/basic-tags-cleaning/expected.html ================================================

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

================================================ FILE: test/test-pages/basic-tags-cleaning/source.html ================================================ Basic tag cleaning test

Lorem

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Foo

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

================================================ FILE: test/test-pages/bbc-1/expected-metadata.json ================================================ { "title": "Obama admits US gun laws are his 'biggest frustration'", "byline": "BBC News", "dir": null, "lang": "en", "excerpt": "President Barack Obama tells the BBC his failure to pass \"common sense gun safety laws\" is the greatest frustration of his presidency.", "siteName": "BBC News", "publishedTime": "2015-07-24T05:36:09+01:00", "readerable": true } ================================================ FILE: test/test-pages/bbc-1/expected.html ================================================

President Barack Obama has admitted that his failure to pass "common sense gun safety laws" in the US is the greatest frustration of his presidency.

In an interview with the BBC, Mr Obama said it was "distressing" not to have made progress on the issue "even in the face of repeated mass killings".

He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president did not sound very confident.

However, Mr Obama said race relations had improved during his presidency.

Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana, killing two people and injuring several others before shooting himself.

In a wide-ranging interview, President Obama also said:

  • The UK must stay in the EU to have influence on the world stage
  • He is confident the Iran nuclear deal will be passed by Congress
  • Syria needs a political solution in order to defeat the Islamic State group
  • He would speak "bluntly" against corruption and human rights violations in Kenya
  • He would defend his advocacy of gay rights following protests in Kenya
  • Despite racial tensions, the US is becoming more diverse and more tolerant

Read the full transcript of his interview

Mr Obama lands in Kenya later on Friday for his first visit since becoming president.

But with just 18 months left in power, he said gun control was the area where he has been "most frustrated and most stymied" since coming to power in 2009.

"If you look at the number of Americans killed since 9/11 by terrorism, it's less than 100. If you look at the number that have been killed by gun violence, it's in the tens of thousands," Mr Obama said.

Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013
The president said he would continue fighting for greater gun control laws

"For us not to be able to resolve that issue has been something that is distressing," he added.

Mr Obama has pushed for stricter gun control throughout his presidency but has been unable to secure any significant changes to the laws.

After nine African-American churchgoers were killed in South Carolina in June, he admitted "politics in this town" meant there were few options available.

line

Analysis: Jon Sopel, BBC News, Washington

President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015

Nine months ago, the president seemed like a spent force, after taking a beating in the midterm elections, during which members of his own party were reluctant to campaign on his record.

But the man sat before me today was relaxed and confident, buoyed by a string of "wins" on healthcare, Cuba and Iran, after bitter and ongoing battles with his many critics.

The only body swerve the president performed was when I asked him how many minds he had changed on the Iran nuclear deal after an intense sell aimed at Gulf allies and members of US Congress who remain implacably opposed.

There was a momentary flicker across the president's face as if to say "You think you got me?" before his smile returned and he proceeded to talk about how Congress would come round.

But notably, he did not give a direct answer to that question, which leaves me with the impression that he has persuaded precisely zero.

Five things we learned from Obama interview

The presidential body swerve

line

On race relations, Mr Obama said recent concerns around policing and mass incarcerations were "legitimate and deserve intense attention" but insisted progress had been made.

Children growing up during the eight years of his presidency "will have a different view of race relations in this country and what's possible," he said.

"There are going to be tensions that arise. But if you look at my daughters' generation, they have an attitude about race that's entirely different than even my generation."

Talking about how he was feeling after his recent successes, he said "every president, every leader has strengths and weaknesses".

"One of my strengths is I have a pretty even temperament. I don't get too high when it's high and I don't get too low when it's low," he said.

Customer looks at Obama shirts at a stall in Nairobi's Kibera slums, 23 July 2015
Kenya is getting ready to welcome the US president

Kenya trip

Mr Obama was speaking to the BBC at the White House before departing for Kenya.

His father was Kenyan and the president is expected to meet relatives in Nairobi.

Mr Obama has faced criticism in the country after the US legalised gay marriage. However, in his interview, the president said he would not fall silent on the issue.

"I am not a fan of discrimination and bullying of anybody on the basis of race, on the basis of religion, on the basis of sexual orientation or gender," he said.

The president also admitted that some African governments, including Kenya's, needed to improve their records on human rights and democracy. However, he defended his decision to engage with and visit those governments.

"Well, they're not ideal institutions. But what we found is, is that when we combined blunt talk with engagement, that gives us the best opportunity to influence and open up space for civil society."

Mr Obama will become the first US president to address the African Union when he travels on to Ethiopia on Sunday.

================================================ FILE: test/test-pages/bbc-1/source.html ================================================ Obama admits US gun laws are his 'biggest frustration' - BBC News
US & Canada
US & Canada

Obama admits US gun laws are his 'biggest frustration'

  • 24 July 2015
  • From the section US & Canada
  • 941 comments
Media caption Mr Obama told the BBC that gun control was his biggest frustration

President Barack Obama has admitted that his failure to pass "common sense gun safety laws" in the US is the greatest frustration of his presidency.

In an interview with the BBC, Mr Obama said it was "distressing" not to have made progress on the issue "even in the face of repeated mass killings".

He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president did not sound very confident.

However, Mr Obama said race relations had improved during his presidency.

Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana, killing two people and injuring several others before shooting himself.

In a wide-ranging interview, President Obama also said:

  • The UK must stay in the EU to have influence on the world stage
  • He is confident the Iran nuclear deal will be passed by Congress
  • Syria needs a political solution in order to defeat the Islamic State group
  • He would speak "bluntly" against corruption and human rights violations in Kenya
  • He would defend his advocacy of gay rights following protests in Kenya
  • Despite racial tensions, the US is becoming more diverse and more tolerant

Read the full transcript of his interview

Mr Obama lands in Kenya later on Friday for his first visit since becoming president.

But with just 18 months left in power, he said gun control was the area where he has been "most frustrated and most stymied" since coming to power in 2009.

"If you look at the number of Americans killed since 9/11 by terrorism, it's less than 100. If you look at the number that have been killed by gun violence, it's in the tens of thousands," Mr Obama said.

Media caption Barack Obama: "Great Britain has always been our best partner"
Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013
The president said he would continue fighting for greater gun control laws

"For us not to be able to resolve that issue has been something that is distressing," he added.

Mr Obama has pushed for stricter gun control throughout his presidency but has been unable to secure any significant changes to the laws.

After nine African-American churchgoers were killed in South Carolina in June, he admitted "politics in this town" meant there were few options available.

line

Analysis: Jon Sopel, BBC News, Washington

President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015

Nine months ago, the president seemed like a spent force, after taking a beating in the midterm elections, during which members of his own party were reluctant to campaign on his record.

But the man sat before me today was relaxed and confident, buoyed by a string of "wins" on healthcare, Cuba and Iran, after bitter and ongoing battles with his many critics.

The only body swerve the president performed was when I asked him how many minds he had changed on the Iran nuclear deal after an intense sell aimed at Gulf allies and members of US Congress who remain implacably opposed.

There was a momentary flicker across the president's face as if to say "You think you got me?" before his smile returned and he proceeded to talk about how Congress would come round.

But notably, he did not give a direct answer to that question, which leaves me with the impression that he has persuaded precisely zero.

Five things we learned from Obama interview

The presidential body swerve

line
Media caption The BBC contrasts President Obama's reactions after mass shootings, with the levels of US gun ownership during his terms in office. (Video by David Botti)

On race relations, Mr Obama said recent concerns around policing and mass incarcerations were "legitimate and deserve intense attention" but insisted progress had been made.

Children growing up during the eight years of his presidency "will have a different view of race relations in this country and what's possible," he said.

"There are going to be tensions that arise. But if you look at my daughters' generation, they have an attitude about race that's entirely different than even my generation."

Talking about how he was feeling after his recent successes, he said "every president, every leader has strengths and weaknesses".

"One of my strengths is I have a pretty even temperament. I don't get too high when it's high and I don't get too low when it's low," he said.

Customer looks at Obama shirts at a stall in Nairobi's Kibera slums, 23 July 2015
Kenya is getting ready to welcome the US president

Kenya trip

Mr Obama was speaking to the BBC at the White House before departing for Kenya.

His father was Kenyan and the president is expected to meet relatives in Nairobi.

Mr Obama has faced criticism in the country after the US legalised gay marriage. However, in his interview, the president said he would not fall silent on the issue.

Media caption President Obama told the BBC he would deliver a blunt message on gay rights when he travelled to Africa

"I am not a fan of discrimination and bullying of anybody on the basis of race, on the basis of religion, on the basis of sexual orientation or gender," he said.

The president also admitted that some African governments, including Kenya's, needed to improve their records on human rights and democracy. However, he defended his decision to engage with and visit those governments.

"Well, they're not ideal institutions. But what we found is, is that when we combined blunt talk with engagement, that gives us the best opportunity to influence and open up space for civil society."

Mr Obama will become the first US president to address the African Union when he travels on to Ethiopia on Sunday.

More on this story

News navigation

Krux('scrape', { 'page_attr_description': {meta_name: 'Description'}});
================================================ FILE: test/test-pages/blogger/expected-metadata.json ================================================ { "title": "Open Verilog flow for Silego GreenPak4 programmable logic devices", "byline": null, "dir": "ltr", "excerpt": "I've written a couple of posts in the past few months but they were all for the blog at work so I figured I'm long overdue for one on Silic...", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/blogger/expected.html ================================================

I've written a couple of posts in the past few months but they were all for the blog at work so I figured I'm long overdue for one on Silicon Exposed.

So what's a GreenPak?

Silego Technology is a fabless semiconductor company located in the SF Bay area, which makes (among other things) a line of programmable logic devices known as GreenPak. Their 5th generation parts were just announced, but I started this project before that happened so I'm still targeting the 4th generation.

GreenPak devices are kind of like itty bitty PSoCs - they have a mixed signal fabric with an ADC, DACs, comparators, voltage references, plus a digital LUT/FF fabric and some typical digital MCU peripherals like counters and oscillators (but no CPU).

It's actually an interesting architecture - FPGAs (including some devices marketed as CPLDs) are a 2D array of LUTs connected via wires to adjacent cells, and true (product term) CPLDs are a star topology of AND-OR arrays connected by a crossbar. GreenPak, on the other hand, is a star topology of LUTs, flipflops, and analog/digital hard IP connected to a crossbar.

Without further ado, here's a block diagram showing all the cool stuff you get in the SLG46620V:

SLG46620V block diagram (from device datasheet)

They're also tiny (the SLG46620V is a 20-pin 0.4mm pitch STQFN measuring 2x3 mm, and the lower gate count SLG46140V is a mere 1.6x2 mm) and probably the cheapest programmable logic device on the market - $0.50 in low volume and less than $0.40 in larger quantities.

The Vdd range of GreenPak4 is huge, more like what you'd expect from an MCU than an FPGA! It can run on anything from 1.8 to 5V, although performance is only specified at 1.8, 3.3, and 5V nominal voltages. There's also a dual-rail version that trades one of the GPIO pins for a second power supply pin, allowing you to interface to logic at two different voltage levels.

To support low-cost/space-constrained applications, they even have the configuration memory on die. It's one-time programmable and needs external Vpp to program (presumably Silego didn't want to waste die area on charge pumps that would only be used once) but has a SRAM programming mode for prototyping.

The best part is that the development software (GreenPak Designer) is free of charge and provided for all major operating systems including Linux! Unfortunately, the only supported design entry method is schematic entry and there's no way to write your design in a HDL.

While schematics may be fine for quick tinkering on really simple designs, they quickly get unwieldy. The nightmare of a circuit shown below is just a bunch of counters hooked up to LEDs that blink at various rates.

Schematic from hell!

As if this wasn't enough of a problem, the largest GreenPak4 device (the SLG46620V) is split into two halves with limited routing between them, and the GUI doesn't help the user manage this complexity at all - you have to draw your schematic in two halves and add "cross connections" between them.

The icing on the cake is that schematics are a pain to diff and collaborate on. Although GreenPak schematics are XML based, which is a touch better than binary, who wants to read a giant XML diff and try to figure out what's going on in the circuit?

This isn't going to be a post on the quirks of Silego's software, though - that would be boring. As it turns out, there's one more exciting feature of these chips that I didn't mention earlier: the configuration bitstream is 100% documented in the device datasheet! This is unheard of in the programmable logic world. As Nick of Arachnid Labs says, the chip is "just dying for someone to write a VHDL or Verilog compiler for it". As you can probably guess by from the title of this post, I've been busy doing exactly that.

Great! How does it work?

Rather than wasting time writing a synthesizer, I decided to write a GreenPak technology library for Clifford Wolf's excellent open source synthesis tool, Yosys, and then make a place-and-route tool to turn that into a final netlist. The post-PAR netlist can then be loaded into GreenPak Designer in order to program the device.

The first step of the process is to run the "synth_greenpak4" Yosys flow on the Verilog source. This runs a generic RTL synthesis pass, then some coarse-grained extraction passes to infer shift register and counter cells from behavioral logic, and finally maps the remaining logic to LUT/FF cells and outputs a JSON-formatted netlist.

Once the design has been synthesized, my tool (named, surprisingly, gp4par) is then launched on the netlist. It begins by parsing the JSON and constructing a directed graph of cell objects in memory. A second graph, containing all of the primitives in the device and the legal connections between them, is then created based on the device specified on the command line. (As of now only the SLG46620V is supported; the SLG46621V can be added fairly easily but the SLG46140V has a slightly different microarchitecture which will require a bit more work to support.)

After the graphs are generated, each node in the netlist graph is assigned a numeric label identifying the type of cell and each node in the device graph is assigned a list of legal labels: for example, an I/O buffer site is legal for an input buffer, output buffer, or bidirectional buffer.

Example labeling for a subset of the netlist and device graphs

The labeled nodes now need to be placed. The initial placement uses a simple greedy algorithm to create a valid (although not necessarily optimal or even routable) placement:

  1. Loop over the cells in the netlist. If any cell has a LOC constraint, which locks the cell to a specific physical site, attempt to assign the node to the specified site. If the specified node is the wrong type, doesn't exist, or is already used by another constrained node, the constraint is invalid so fail with an error.
  2. Loop over all of the unconstrained cells in the netlist and assign them to the first unused site with the right label. If none are available, the design is too big for the device so fail with an error.

Once the design is placed, the placement optimizer then loops over the design and attempts to improve it. A simulated annealing algorithm is used, where changes to the design are accepted unconditionally if they make the placement better, and with a random, gradually decreasing probability if they make it worse. The optimizer terminates when the design receives a perfect score (indicating an optimal placement) or if it stops making progress for several iterations. Each iteration does the following:

  1. Compute a score for the current design based on the number of unroutable nets, the amount of routing congestion (number of nets crossing between halves of the device), and static timing analysis (not yet implemented, always zero).
  2. Make a list of nodes that contributed to this score in some way (having some attached nets unroutable, crossing to the other half of the device, or failing timing).
  3. Remove nodes from the list that are LOC'd to a specific location since we're not allowed to move them.
  4. Remove nodes from the list that have only one legal placement in the device (for example, oscillator hard IP) since there's nowhere else for them to go.
  5. Pick a node from the remainder of the list at random. Call this our pivot.
  6. Find a list of candidate placements for the pivot:
    1. Consider all routable placements in the other half of the device.
    2. If none were found, consider all routable placements anywhere in the device.
    3. If none were found, consider all placements anywhere in the device even if they're not routable.
  7. Pick one of the candidates at random and move the pivot to that location. If another cell in the netlist is already there, put it in the vacant site left by the pivot.
  8. Re-compute the score for the design. If it's better, accept this change and start the next iteration.
  9. If the score is worse, accept it with a random probability which decreases as the iteration number goes up. If the change is not accepted, restore the previous placement.

After optimization, the design is checked for routability. If any edges in the netlist graph don't correspond to edges in the device graph, the user probably asked for something impossible (for example, trying to hook a flipflop's output to a comparator's reference voltage input) so fail with an error.

The design is then routed. This is quite simple due to the crossbar structure of the device. For each edge in the netlist:

  1. If dedicated (non-fabric) routing is used for this path, configure the destination's input mux appropriately and stop.
  2. If the source and destination are in the same half of the device, configure the destination's input mux appropriately and stop.
  3. A cross-connection must be used. Check if we already used one to bring the source signal to the other half of the device. If found, configure the destination to route from that cross-connection and stop.
  4. Check if we have any cross-connections left going in this direction. If they're all used, the design is unroutable due to congestion so fail with an error.
  5. Pick the next unused cross-connection and configure it to route from the source. Configure the destination to route from the cross-connection and stop.

Once routing is finished, run a series of post-PAR design rule checks. These currently include the following:

  • If any node has no loads, generate a warning
  • If an I/O buffer is connected to analog hard IP, fail with an error if it's not configured in analog mode.
  • Some signals (such as comparator inputs and oscillator power-down controls) are generated by a shared mux and fed to many loads. If different loads require conflicting settings for the shared mux, fail with an error.

If DRC passes with no errors, configure all of the individual cells in the netlist based on the HDL parameters. Fail with an error if an invalid configuration was requested.

Finally, generate the bitstream from all of the per-cell configuration and write it to a file.

Great, let's get started!

If you don't already have one, you'll need to buy a GreenPak4 development kit. The kit includes samples of the SLG46620V (among other devices) and a programmer/emulation board. While you're waiting for it to arrive, install GreenPak Designer.

Download and install Yosys. Although Clifford is pretty good at merging my pull requests, only my fork on Github is guaranteed to have the most up-to-date support for GreenPak devices so don't be surprised if you can't use a bleeding-edge feature with mainline Yosys.

Download and install gp4par. You can get it from the Github repository.

Write your HDL, compile with Yosys, P&R with gp4par, and import the bitstream into GreenPak Designer to program the target device. The most current gp4par manual is included in LaTeX source form in the source tree and is automatically built as part of the compile process. If you're just browsing, there's a relatively recent PDF version on my web server.

If you'd like to see the Verilog that produced the nightmare of a schematic I showed above, here it is.

Be advised that this project is still very much a work in progress and there are still a number of SLG46620V features I don't support (see the manual for exact details).

I love it / it segfaulted / there's a problem in the manual!

Hop in our IRC channel (##openfpga on Freenode) and let me know. Feedback is great, pull requests are even better,

You're competing with Silego's IDE. Have they found out and sued you yet?

Nope. They're fully aware of what I'm doing and are rolling out the red carpet for me. They love the idea of a HDL flow as an alternative to schematic entry and are pretty amazed at how fast it's coming together.

After I reported a few bugs in their datasheets they decided to skip the middleman and give me direct access to the engineer who writes their documentation so that I can get faster responses. The last time I found a problem (two different parts of the datasheet contradicted each other) an updated datasheet was in my inbox and on their website by the next day. I only wish Xilinx gave me that kind of treatment!

They've even offered me free hardware to help me add support for their latest product family, although I plan to get GreenPak4 support to a more stable state before taking them up on the offer.

So what's next?

Better testing, for starters. I have to verify functionality by hand with a DMM and oscilloscope, which is time consuming.

My contact at Silego says they're going to be giving me documentation on the SRAM emulation interface soon, so I'm going to make a hardware-in-loop test platform that connects to my desktop and the Silego ZIF socket, and lets me load new bitstreams via a scriptable interface. It'll have FPGA-based digital I/O as well as an ADC and DAC on every device pin, plus an adjustable voltage regulator for power, so I can feed in arbitrary mixed-signal test waveforms and write PC-based unit tests to verify correct behavior.

Other than that, I want to finish support for the SLG46620V in the next month or two. The SLG46621V will be an easy addition since only one pin and the relevant configuration bits have changed from the 46620 (I suspect they're the same die, just bonded out differently).

Once that's done I'll have to do some more extensive work to add the SLG46140V since the architecture is a bit different (a lot of the combinatorial logic is merged into multi-function blocks). Luckily, the 46140 has a lot in common architecturally with the GreenPak5 family, so once that's done GreenPak5 will probably be a lot easier to add support for.

My thanks go out to Clifford Wolf, whitequark, the IRC users in ##openfpga, and everyone at Silego I've worked with to help make this possible. I hope that one day this project will become mature enough that Silego will ship it as an officially supported extension to GreenPak Designer, making history by becoming the first modern programmable logic vendor to ship a fully open source synthesis and P&R suite.

================================================ FILE: test/test-pages/blogger/source.html ================================================ Silicon Exposed: Open Verilog flow for Silego GreenPak4 programmable logic devices

Sunday, May 8, 2016

Open Verilog flow for Silego GreenPak4 programmable logic devices

I've written a couple of posts in the past few months but they were all for the blog at work so I figured I'm long overdue for one on Silicon Exposed.

So what's a GreenPak?


Silego Technology is a fabless semiconductor company located in the SF Bay area, which makes (among other things) a line of programmable logic devices known as GreenPak. Their 5th generation parts were just announced, but I started this project before that happened so I'm still targeting the 4th generation.

GreenPak devices are kind of like itty bitty PSoCs - they have a mixed signal fabric with an ADC, DACs, comparators, voltage references, plus a digital LUT/FF fabric and some typical digital MCU peripherals like counters and oscillators (but no CPU).

It's actually an interesting architecture - FPGAs (including some devices marketed as CPLDs) are a 2D array of LUTs connected via wires to adjacent cells, and true (product term) CPLDs are a star topology of AND-OR arrays connected by a crossbar. GreenPak, on the other hand, is a star topology of LUTs, flipflops, and analog/digital hard IP connected to a crossbar.

Without further ado, here's a block diagram showing all the cool stuff you get in the SLG46620V:

SLG46620V block diagram (from device datasheet)
They're also tiny (the SLG46620V is a 20-pin 0.4mm pitch STQFN measuring 2x3 mm, and the lower gate count SLG46140V is a mere 1.6x2 mm) and probably the cheapest programmable logic device on the market - $0.50 in low volume and less than $0.40 in larger quantities.

The Vdd range of GreenPak4 is huge, more like what you'd expect from an MCU than an FPGA! It can run on anything from 1.8 to 5V, although performance is only specified at 1.8, 3.3, and 5V nominal voltages. There's also a dual-rail version that trades one of the GPIO pins for a second power supply pin, allowing you to interface to logic at two different voltage levels.

To support low-cost/space-constrained applications, they even have the configuration memory on die. It's one-time programmable and needs external Vpp to program (presumably Silego didn't want to waste die area on charge pumps that would only be used once) but has a SRAM programming mode for prototyping.

The best part is that the development software (GreenPak Designer) is free of charge and provided for all major operating systems including Linux! Unfortunately, the only supported design entry method is schematic entry and there's no way to write your design in a HDL.

While schematics may be fine for quick tinkering on really simple designs, they quickly get unwieldy. The nightmare of a circuit shown below is just a bunch of counters hooked up to LEDs that blink at various rates.

Schematic from hell!
As if this wasn't enough of a problem, the largest GreenPak4 device (the SLG46620V) is split into two halves with limited routing between them, and the GUI doesn't help the user manage this complexity at all - you have to draw your schematic in two halves and add "cross connections" between them.

The icing on the cake is that schematics are a pain to diff and collaborate on. Although GreenPak schematics are XML based, which is a touch better than binary, who wants to read a giant XML diff and try to figure out what's going on in the circuit?

This isn't going to be a post on the quirks of Silego's software, though - that would be boring. As it turns out, there's one more exciting feature of these chips that I didn't mention earlier: the configuration bitstream is 100% documented in the device datasheet! This is unheard of in the programmable logic world. As Nick of Arachnid Labs says, the chip is "just dying for someone to write a VHDL or Verilog compiler for it". As you can probably guess by from the title of this post, I've been busy doing exactly that.

Great! How does it work?


Rather than wasting time writing a synthesizer, I decided to write a GreenPak technology library for Clifford Wolf's excellent open source synthesis tool, Yosys, and then make a place-and-route tool to turn that into a final netlist. The post-PAR netlist can then be loaded into GreenPak Designer in order to program the device.

The first step of the process is to run the "synth_greenpak4" Yosys flow on the Verilog source. This runs a generic RTL synthesis pass, then some coarse-grained extraction passes to infer shift register and counter cells from behavioral logic, and finally maps the remaining logic to LUT/FF cells and outputs a JSON-formatted netlist.

Once the design has been synthesized, my tool (named, surprisingly, gp4par) is then launched on the netlist. It begins by parsing the JSON and constructing a directed graph of cell objects in memory. A second graph, containing all of the primitives in the device and the legal connections between them, is then created based on the device specified on the command line. (As of now only the SLG46620V is supported; the SLG46621V can be added fairly easily but the SLG46140V has a slightly different microarchitecture which will require a bit more work to support.)

After the graphs are generated, each node in the netlist graph is assigned a numeric label identifying the type of cell and each node in the device graph is assigned a list of legal labels: for example, an I/O buffer site is legal for an input buffer, output buffer, or bidirectional buffer.

Example labeling for a subset of the netlist and device graphs
The labeled nodes now need to be placed. The initial placement uses a simple greedy algorithm to create a valid (although not necessarily optimal or even routable) placement:
  1. Loop over the cells in the netlist. If any cell has a LOC constraint, which locks the cell to a specific physical site, attempt to assign the node to the specified site. If the specified node is the wrong type, doesn't exist, or is already used by another constrained node, the constraint is invalid so fail with an error.
  2. Loop over all of the unconstrained cells in the netlist and assign them to the first unused site with the right label. If none are available, the design is too big for the device so fail with an error.
Once the design is placed, the placement optimizer then loops over the design and attempts to improve it. A simulated annealing algorithm is used, where changes to the design are accepted unconditionally if they make the placement better, and with a random, gradually decreasing probability if they make it worse. The optimizer terminates when the design receives a perfect score (indicating an optimal placement) or if it stops making progress for several iterations. Each iteration does the following:
  1. Compute a score for the current design based on the number of unroutable nets, the amount of routing congestion (number of nets crossing between halves of the device), and static timing analysis (not yet implemented, always zero).
  2. Make a list of nodes that contributed to this score in some way (having some attached nets unroutable, crossing to the other half of the device, or failing timing).
  3. Remove nodes from the list that are LOC'd to a specific location since we're not allowed to move them.
  4. Remove nodes from the list that have only one legal placement in the device (for example, oscillator hard IP) since there's nowhere else for them to go.
  5. Pick a node from the remainder of the list at random. Call this our pivot.
  6. Find a list of candidate placements for the pivot:
    1. Consider all routable placements in the other half of the device.
    2. If none were found, consider all routable placements anywhere in the device.
    3. If none were found, consider all placements anywhere in the device even if they're not routable.
  7. Pick one of the candidates at random and move the pivot to that location. If another cell in the netlist is already there, put it in the vacant site left by the pivot.
  8. Re-compute the score for the design. If it's better, accept this change and start the next iteration.
  9. If the score is worse, accept it with a random probability which decreases as the iteration number goes up. If the change is not accepted, restore the previous placement.
After optimization, the design is checked for routability. If any edges in the netlist graph don't correspond to edges in the device graph, the user probably asked for something impossible (for example, trying to hook a flipflop's output to a comparator's reference voltage input) so fail with an error.

The design is then routed. This is quite simple due to the crossbar structure of the device. For each edge in the netlist:
  1. If dedicated (non-fabric) routing is used for this path, configure the destination's input mux appropriately and stop.
  2. If the source and destination are in the same half of the device, configure the destination's input mux appropriately and stop.
  3. A cross-connection must be used. Check if we already used one to bring the source signal to the other half of the device. If found, configure the destination to route from that cross-connection and stop.
  4. Check if we have any cross-connections left going in this direction. If they're all used, the design is unroutable due to congestion so fail with an error.
  5. Pick the next unused cross-connection and configure it to route from the source. Configure the destination to route from the cross-connection and stop.
Once routing is finished, run a series of post-PAR design rule checks. These currently include the following:
  • If any node has no loads, generate a warning
  • If an I/O buffer is connected to analog hard IP, fail with an error if it's not configured in analog mode.
  • Some signals (such as comparator inputs and oscillator power-down controls) are generated by a shared mux and fed to many loads. If different loads require conflicting settings for the shared mux, fail with an error.
If DRC passes with no errors, configure all of the individual cells in the netlist based on the HDL parameters. Fail with an error if an invalid configuration was requested.

Finally, generate the bitstream from all of the per-cell configuration and write it to a file.

Great, let's get started!

If you don't already have one, you'll need to buy a GreenPak4 development kit. The kit includes samples of the SLG46620V (among other devices) and a programmer/emulation board. While you're waiting for it to arrive, install GreenPak Designer.

Download and install Yosys. Although Clifford is pretty good at merging my pull requests, only my fork on Github is guaranteed to have the most up-to-date support for GreenPak devices so don't be surprised if you can't use a bleeding-edge feature with mainline Yosys.

Download and install gp4par. You can get it from the Github repository.

Write your HDL, compile with Yosys, P&R with gp4par, and import the bitstream into GreenPak Designer to program the target device. The most current gp4par manual is included in LaTeX source form in the source tree and is automatically built as part of the compile process. If you're just browsing, there's a relatively recent PDF version on my web server.

If you'd like to see the Verilog that produced the nightmare of a schematic I showed above, here it is.

Be advised that this project is still very much a work in progress and there are still a number of SLG46620V features I don't support (see the manual for exact details).

I love it / it segfaulted / there's a problem in the manual!

Hop in our IRC channel (##openfpga on Freenode) and let me know. Feedback is great, pull requests are even better,

You're competing with Silego's IDE. Have they found out and sued you yet?

Nope. They're fully aware of what I'm doing and are rolling out the red carpet for me. They love the idea of a HDL flow as an alternative to schematic entry and are pretty amazed at how fast it's coming together.

After I reported a few bugs in their datasheets they decided to skip the middleman and give me direct access to the engineer who writes their documentation so that I can get faster responses. The last time I found a problem (two different parts of the datasheet contradicted each other) an updated datasheet was in my inbox and on their website by the next day. I only wish Xilinx gave me that kind of treatment!

They've even offered me free hardware to help me add support for their latest product family, although I plan to get GreenPak4 support to a more stable state before taking them up on the offer.

So what's next?


Better testing, for starters. I have to verify functionality by hand with a DMM and oscilloscope, which is time consuming.

My contact at Silego says they're going to be giving me documentation on the SRAM emulation interface soon, so I'm going to make a hardware-in-loop test platform that connects to my desktop and the Silego ZIF socket, and lets me load new bitstreams via a scriptable interface. It'll have FPGA-based digital I/O as well as an ADC and DAC on every device pin, plus an adjustable voltage regulator for power, so I can feed in arbitrary mixed-signal test waveforms and write PC-based unit tests to verify correct behavior.

Other than that, I want to finish support for the SLG46620V in the next month or two. The SLG46621V will be an easy addition since only one pin and the relevant configuration bits have changed from the 46620 (I suspect they're the same die, just bonded out differently).

Once that's done I'll have to do some more extensive work to add the SLG46140V since the architecture is a bit different (a lot of the combinatorial logic is merged into multi-function blocks). Luckily, the 46140 has a lot in common architecturally with the GreenPak5 family, so once that's done GreenPak5 will probably be a lot easier to add support for.

My thanks go out to Clifford Wolf, whitequark, the IRC users in ##openfpga, and everyone at Silego I've worked with to help make this possible. I hope that one day this project will become mature enough that Silego will ship it as an officially supported extension to GreenPak Designer, making history by becoming the first modern programmable logic vendor to ship a fully open source synthesis and P&R suite.
Posted by at
Labels: ,

No comments:

Post a Comment

Subscribe to: Post Comments (Atom)
================================================ FILE: test/test-pages/breitbart/expected-metadata.json ================================================ { "title": "'Neutral' Snopes Fact-Checker David Emery: 'Are There Any Un-Angry Trump Supporters?' - Breitbart", "byline": "by Lucas Nolan22 Dec 2016651", "dir": "ltr", "lang": "en", "excerpt": "Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”", "siteName": "Breitbart", "publishedTime": "2016-12-22T10:43:37-08:00", "readerable": true } ================================================ FILE: test/test-pages/breitbart/expected.html ================================================

Supporters of Republican presidential nominee Donald Trump cheer during election night at the New York Hilton Midtown in New York on November 9, 2016. / AFP / JIM WATSON (Photo credit should read JIM WATSON/AFP/Getty Images)

JIM WATSON/AFP/Getty Images

SIGN UP FOR OUR NEWSLETTER

Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”

Emery, a writer for partisan “fact-checking” website Snopes.com which soon will be in charge of labelling “fake news” alongside ABC News and Politifact, retweeted an article by Vulture magazine relating to the protests of the Hamilton musical following the decision by the cast of the show to make a public announcement to Vice-president elect Mike Pence while he watched the performance with his family.

SIGN UP FOR OUR NEWSLETTER

The tweet from Vulture magazine reads, “#Hamilton Chicago show interrupted by angry Trump supporter.” Emery retweeted the story, saying, “Are there un-angry Trump supporters?”

This isn’t the first time the Snopes.com writer has expressed anti-Trump sentiment on his Twitter page. In another tweet in which Emery links to an article that falsely attributes a quote to President-elect Trump, Emery states, “Incredibly, some people actually think they have to put words in Trump’s mouth to make him look bad.”

Emery also retweeted an article by New York magazine that claimed President-elect Trump relied on lies to win during his campaign and that we now lived in a “post-truth” society. “Before long we’ll all have forgotten what it was like to live in the same universe; or maybe we already have,” Emery tweeted.

Facebook believe that Emery, along with other Snopes writers, ABC News, and Politifact are impartial enough to label and silence what they believe to be “fake news” on social media.

Lucas Nolan is a reporter for Breitbart Tech covering issues of free speech and online censorship. Follow him on Twitter @LucasNolan_ or email him at lnolan@breitbart.com

================================================ FILE: test/test-pages/breitbart/source.html ================================================ 'Neutral' Snopes Fact-Checker David Emery: 'Are There Any Un-Angry Trump Supporters?' - Breitbart
Skip to content
report this ad

'Neutral' Snopes Fact-Checker David Emery: 'Are There Any Un-Angry Trump Supporters?'

Supporters of Republican presidential nominee Donald Trump cheer during election night at the New York Hilton Midtown in New York on November 9, 2016. / AFP / JIM WATSON (Photo credit should read JIM WATSON/AFP/Getty Images)
JIM WATSON/AFP/Getty Images

by Lucas Nolan22 Dec 2016651

SIGN UP FOR OUR NEWSLETTER

Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”

Emery, a writer for partisan “fact-checking” website Snopes.com which soon will be in charge of labelling “fake news” alongside ABC News and Politifact, retweeted an article by Vulture magazine relating to the protests of the Hamilton musical following the decision by the cast of the show to make a public announcement to Vice-president elect Mike Pence while he watched the performance with his family.

SIGN UP FOR OUR NEWSLETTER

The tweet from Vulture magazine reads, “#Hamilton Chicago show interrupted by angry Trump supporter.” Emery retweeted the story, saying, “Are there un-angry Trump supporters?”

This isn’t the first time the Snopes.com writer has expressed anti-Trump sentiment on his Twitter page. In another tweet in which Emery links to an article that falsely attributes a quote to President-elect Trump, Emery states, “Incredibly, some people actually think they have to put words in Trump’s mouth to make him look bad.”

Emery also retweeted an article by New York magazine that claimed President-elect Trump relied on lies to win during his campaign and that we now lived in a “post-truth” society. “Before long we’ll all have forgotten what it was like to live in the same universe; or maybe we already have,” Emery tweeted.

Facebook believe that Emery, along with other Snopes writers, ABC News, and Politifact are impartial enough to label and silence what they believe to be “fake news” on social media.

Lucas Nolan is a reporter for Breitbart Tech covering issues of free speech and online censorship. Follow him on Twitter @LucasNolan_ or email him at lnolan@breitbart.com

report this ad
Lumify Flashlight X9
Best Flashlight Ever is Selling Like CrazyLumify Flashlight X9
Undo
My Snoring Solution
Say Goodbye To Snoring, Say Hello to SleepMy Snoring Solution
Undo
tripsinsider.com
Finding Cheap Flights is Now Easytripsinsider.com
Undo
LifeDaily.com
Fresh Evidence In Maddie McCann Mystery Stuns PoliceLifeDaily.com
Undo
Smart Device Trends
Finally You Can Track Your Car Using Your SmartphoneSmart Device Trends
Undo
Womensarticle.com
5 College Degrees That Will Be Extinct In 20 YearsWomensarticle.com
Undo
Carophile
56 of the Worst Cars of All TimeCarophile
Undo
My Snoring Solution
End Your Nightly Snoring Nightmare With This Simple SolutionMy Snoring Solution
Undo
Lumify X9
Crazy Cheap Flashlight - Selling Like Hotcakes!Lumify X9
Undo
LifeDaily.com
Cop Finds Baby In Dumpster: But 25 Years Later, This Happened...LifeDaily.com
Undo
LifeDaily.com
New Evidence In Missing Alabama Teen Mystery Stuns PoliceLifeDaily.com
Undo
LifeDaily.com
Teen Vanishes On Spring Break: But 7 Years Later, Police Uncover Truth...LifeDaily.com
Undo
LifeDaily.com
Couple Finds 50-Year-Old Fallout Shelter, But When They Look inside...LifeDaily.com
Undo
LifeDaily.com
Baby Girl Vanishes From Crib: But 20 Years Later, This Happened...LifeDaily.com
Undo
LifeDaily.com
Mysterious Alcatraz Escape Unsolved For Years, Until Police Find This...LifeDaily.com
Undo
My Snoring Solution
A Solution That Puts Snoring to BedMy Snoring Solution
Undo
LifeDaily.com
Missing Teens Unsolved For 15 Years, Until Police Uncover Truth...LifeDaily.com
Undo
Womensarticle.com
10 Shockingly High Paying Graduate College DegreesWomensarticle.com
Undo
Trending Articles

Gloria Steinem: Woman Felt ‘Sexually Assaulted’ by Trump’s…

Monday on MSNBC’s “All In,” discussing Saturday’s women’s march on Washington, feminist and…

Powered By
Around The Web Powered By ZergNet
Major Break in the D.B. Cooper MysteryNypost.com
What Trump Fans Think of Megyn Kelly's New GigAol.com
The Real Reason Sasha Was Absent From Her Dad's Farewell SpeechUsmagazine.com


Comment count on this article reflects comments made on Breitbart.com and Facebook. Visit Breitbart's Facebook Page.

AddThis Sharing
­ ================================================ FILE: test/test-pages/bug-1255978/expected-metadata.json ================================================ { "title": "Seven secrets that hotel owners don't want you to know", "byline": "Hazel Sheffield", "dir": null, "excerpt": "Most people go to hotels for the pleasure of sleeping in a giant bed with clean white sheets and waking up to fresh towels in the morning. But those towels and sheets might not be as clean as they look, according to the hotel bosses that responded to an online thread about the things hotel owners don’t want you to know.", "siteName": "The Independent", "publishedTime": "2015-09-17T16:57:43+01:00", "readerable": true } ================================================ FILE: test/test-pages/bug-1255978/expected.html ================================================

Most people go to hotels for the pleasure of sleeping in a giant bed with clean white sheets and waking up to fresh towels in the morning.

But those towels and sheets might not be as clean as they look, according to the hotel bosses that responded to an online thread about the things hotel owners don’t want you to know.

Zeev Sharon and Michael Forrest Jones both run hotel start-ups in the US. Forrest Jones runs the start-up Beechmont Hotels Corporation, a hotel operating company that consults with hotel owners on how they can improve their business. Sharon is the CEO of Hotelied, a start-up that allows people to sign up for discounts at luxury hotels.

But even luxury hotels aren’t always cleaned as often as they should be.

Here are some of the secrets that the receptionist will never tell you when you check in, according to answers posted on Quora.

bandb2.jpg

Even posh hotels might not wash a blanket in between stays

1. Take any blankets or duvets off the bed

Forrest Jones said that anything that comes into contact with any of the previous guest’s skin should be taken out and washed every time the room is made, but that even the fanciest hotels don’t always do so. "Hotels are getting away from comforters. Blankets are here to stay, however. But some hotels are still hesitant about washing them every day if they think they can get out of it," he said.

Play Video

Play

Current Time 0:00

/

Duration Time 0:00

Loaded: 0%

Progress: 0%

Remaining Time -0:00

Share

Fullscreen

Video shows bed bug infestation at New York hotel

hotel-door-getty.jpg

Forrest Jones advised stuffing the peep hole with a strip of rolled up notepaper when not in use.

2. Check the peep hole has not been tampered with

This is not common, but can happen, Forrest Jones said. He advised stuffing the peep hole with a strip of rolled up notepaper when not in use. When someone knocks on the door, the paper can be removed to check who is there. If no one is visible, he recommends calling the front desk immediately. “I look forward to the day when I can tell you to choose only hotels where every employee who has access to guestroom keys is subjected to a complete public records background check, prior to hire, and every year or two thereafter. But for now, I can't,” he said.

luggage-3.jpg

Put luggage on the floor

3. Don’t use a wooden luggage rack

Bedbugs love wood. Even though a wooden luggage rack might look nicer and more expensive than a metal one, it’s a breeding ground for bugs. Forrest Jones says guests should put the items they plan to take from bags on other pieces of furniture and leave the bag on the floor.

Lifestyle-hotels.jpg

The old rule of thumb is that for every 00 invested in a room, the hotel should charge in average daily rate

4. Hotel rooms are priced according to how expensive they were to build

Zeev Sharon said that the old rule of thumb is that for every $1000 invested in a room, the hotel should charge $1 in average daily rate. So a room that cost $300,000 to build, should sell on average for $300/night.

5. Beware the wall-mounted hairdryer

It contains the most germs of anything in the room. Other studies have said the TV remote and bedside lamp switches are the most unhygienic. “Perhaps because it's something that's easy for the housekeepers to forget to check or to squirt down with disinfectant,” Forrest Jones said.

6. Mini bars almost always lose money

Despite the snacks in the minibar seeming like the most overpriced food you have ever seen, hotel owners are still struggling to make a profit from those snacks. "Minibars almost always lose money, even when they charge $10 for a Diet Coke,” Sharon said.

agenda7.jpg

Towels should always be cleaned between stays

7. Always made sure the hand towels are clean when you arrive

Forrest Jones made a discovery when he was helping out with the housekeepers. “You know where you almost always find a hand towel in any recently-vacated hotel room that was occupied by a guy? On the floor, next to the bed, about halfway down, maybe a little toward the foot of the bed. Same spot in the floor, next to almost every bed occupied by a man, in every room. I'll leave the rest to your imagination,” he said.

Reuse content

================================================ FILE: test/test-pages/bug-1255978/source.html ================================================ The seven secrets that hotel owners don't want you to know | The Independent
  1. News
  2. Business
  3. Business News

The seven secrets that hotel owners don't want you to know

Hotel bosses spill the beans on cost cutting and horrible hygiene
Click here to receive live updates on the US election
bed-hotel-room.jpg
Even luxury hotels aren’t always cleaned as often as they should be Getty Images

Most people go to hotels for the pleasure of sleeping in a giant bed with clean white sheets and waking up to fresh towels in the morning.

But those towels and sheets might not be as clean as they look, according to the hotel bosses that responded to an online thread about the things hotel owners don’t want you to know.

Zeev Sharon and Michael Forrest Jones both run hotel start-ups in the US. Forrest Jones runs the start-up Beechmont Hotels Corporation, a hotel operating company that consults with hotel owners on how they can improve their business. Sharon is the CEO of Hotelied, a start-up that allows people to sign up for discounts at luxury hotels.

But even luxury hotels aren’t always cleaned as often as they should be.

Here are some of the secrets that the receptionist will never tell you when you check in, according to answers posted on Quora.

bandb2.jpg
Even posh hotels might not wash a blanket in between stays

1. Take any blankets or duvets off the bed

Forrest Jones said that anything that comes into contact with any of the previous guest’s skin should be taken out and washed every time the room is made, but that even the fanciest hotels don’t always do so. "Hotels are getting away from comforters. Blankets are here to stay, however. But some hotels are still hesitant about washing them every day if they think they can get out of it," he said.

Play Video
Play
Mute
Current Time 0:00
/
Duration Time 0:00
Loaded: 0%
Progress: 0%
Stream TypeLIVE
Remaining Time -0:00
Share
Fullscreen
Video shows bed bug infestation at New York hotel

hotel-door-getty.jpg
Forrest Jones advised stuffing the peep hole with a strip of rolled up notepaper when not in use.

2. Check the peep hole has not been tampered with

This is not common, but can happen, Forrest Jones said. He advised stuffing the peep hole with a strip of rolled up notepaper when not in use. When someone knocks on the door, the paper can be removed to check who is there. If no one is visible, he recommends calling the front desk immediately. “I look forward to the day when I can tell you to choose only hotels where every employee who has access to guestroom keys is subjected to a complete public records background check, prior to hire, and every year or two thereafter. But for now, I can't,” he said.

luggage-3.jpg
Put luggage on the floor

3. Don’t use a wooden luggage rack

Bedbugs love wood. Even though a wooden luggage rack might look nicer and more expensive than a metal one, it’s a breeding ground for bugs. Forrest Jones says guests should put the items they plan to take from bags on other pieces of furniture and leave the bag on the floor.

Lifestyle-hotels.jpg
The old rule of thumb is that for every 00 invested in a room, the hotel should charge in average daily rate

4. Hotel rooms are priced according to how expensive they were to build

Zeev Sharon said that the old rule of thumb is that for every $1000 invested in a room, the hotel should charge $1 in average daily rate. So a room that cost $300,000 to build, should sell on average for $300/night.

Read more

5. Beware the wall-mounted hairdryer

It contains the most germs of anything in the room. Other studies have said the TV remote and bedside lamp switches are the most unhygienic. “Perhaps because it's something that's easy for the housekeepers to forget to check or to squirt down with disinfectant,” Forrest Jones said.

Business news in pictures

Business news in pictures

  • 1/50 United goin down

    Manchester United's absence from the Champions League hurt more than the fans' pride last year - it also dented the bottom line. Revenues at the New Uork listed club dipped 8.8 per cent to £395.2m in the year to June, triggering a £1.2 million loss after broadcasting and sponsorship deals dried up. The club said it was now looking to raise $400m from a share issue.

    GETTY IMAGES

  • 2/50 Star Wars boosts economy

    Production of the next Star Wars movie has brought an economic impact of some £150 million to Britain, according to company accounts. The seventh movie in the series, The Force Awakens, will be released in December.

  • 3/50 Natalie Massenet Leaves Net-a-Porter

    The Net-a-Porter founder Natalie Massenet decided to quit the online fashion retailer during "a summer of reflection" that included a spectacular 50th birthday party on the Almalfi coast.

    Rex

  • 4/50 I'll keep working, says Mayer

    The chief executive of Yahoo, Marissa Mayer, said that she was expecting twin girls in December. She said she would "approach the pregnancy and delivery the same way as I did with my son three years ago, taking limited time away and working throughout". In 2012, she took two weeks off when her first child was born.

    GETTY IMAGES

  • 5/50 World's richest lose $182b

    Warren Buffett, the world’s third richest person, lost $3.6bn in last week’s market slump. Fears that China's economy is slowing took many indices into correction territory. The turmoil has wiped $182 billion off the wealth of the 400 richest billionaires, according to research by Bloomberg.

    Getty

  • 6/50 City for sale

    London City Airport was put up for sale by the American hedge fund Global Infrastructure Partners, which also owns Gatwick and Edinburgh airports. It is expected to fetch around £2 billion

    Getty

  • 7/50 Kicking up a skink Down Under

    An Australian court has overturned approval for Adani’s Carmichael coal mine in north Queensland, based on concerns over the impact of the project on the endangered yakka skink (pictured). Standard Chartered is under renewed pressure to stop financing the deal

  • 8/50 Scope to do more in the City?

    Damien Hirst's 7m statue, Charity, has been installed in the shadow of the Gherkin in London. It is based on an old Scope collection from the 1960s and has provoked a debate about the City and disability. The Lord Mayor of London says the City does not employ enough disabled people.

    Reuters/Corbis

  • 9/50 Thousands more take a look at Sky

    Drama and football are a winning formula for Sky, which has recorded its best growth for 11 years, gaining 124,000 UK customers in the past quarter. Sky paid £4.2bn for Premier League rights, but also has 35 dramas planned, following successes such as the ‘The Enfield Haunting’ (pictured) with Timothy Spall.

  • 10/50 FirstGroup suffers over lost lines

    FirstGroup, the bus and rail operator, is paying the price for losing a string of rail franchises last year. Rail revenue will be “substantially lower” in 2015 after some big franchise losses, including ScotRail. The division lost out on tenders for the East Coast and West Coast (pictured) mainlines, which went to Virgin Trains.

    GETTY

  • 11/50 US retail rush hour slows down again

    US retail sales unexpectedly fell in June as households cut purchases of cars and a range of other goods. Sales fell 0.3 per cent, while economists had forecast a small rise.

    AFP

  • 12/50 Blockbusters aid Pinewood expansion

    Pinewood Studios revealed it is having to turn away major films until the five new stages it is building come on stream next summer. The group still saw operating profits rise from £4.9m to £5.8m in the past year, which saw it host the new Star Wars and Avengers films (Scarlett Johansson, pictured, stars in ‘Avengers: Age Of Ultron’) as well as the latest Bond release, ‘Spectre’

    Marvel 2015

  • 13/50 Rogue trader freed

    Kweku Adoboli, the former UBS trader who was convicted of the UK’s biggest banking fraud, has been released from jail after serving half of his seven-year sentence. He was imprisoned in 2012 on two counts of fraud that resulted in losses of £1.4bn, the largest trading loss in British banking history

    Getty

  • 14/50 Burberry’s autumn ad campaign

    Burberry is launching its autumn ad campaign today with a cast of models including the singer Tom Odell and the actress Holliday Grainger

    Burberry/Testino

  • 15/50 Sneak peak at console battlefield

    Videogame titans Microsoft and Sony vied for attention ahead of the industry’s annual E3 conference yesterday, giving fans sneak peaks of the latest games for their Xbox and PlayStation consoles, including Sony’s new ‘Assassin’s Creed Syndicate’

    Getty Images

  • 16/50 High street shops hit by exodus

    The number of Britons out shopping has fallen for the second month in a row. High streets saw 1.5 per cent fewer shoppers and shopping centres 2 per cent fewer in May compared with a year ago. Only out-of-town retail parks saw a rise, with 1.4 per cent more visitors, the British Retail Consortium said. Overall shopper numbers fell by 1 per cent.

    Getty

  • 17/50 Argentina hit by strikes

    Daily life ground to a halt in Buenos Aires yesterday amid a 24-hour general strike protesting against moves to limit pay rises and increase taxes.

    AFP/Getty Images

  • 18/50 Duty rises blamed for beer woes

    Beer sales are still “fragile” after a decade of “devastating” duty hikes, according to the British Beer and Pub Association. The industry lobbyists said sales in pubs and shops were up 1.5 per cent in the past year, but fell by 0.8 per cent in the first quarter. Sales have fallen 24 per cent in the past decade.

  • 19/50 New Look in push for men

    New Look is planning a menswear push after poaching Christopher Englinde, H&M’s menswear chief. Like-for-like sales at the fashion chain rose 5 per cent

    Victoria Middleton

  • 20/50 Court halts Airbus insider trading trial

    A French court has quashed an insider trading case against seven current and former Airbus executives as well as Lagardère and Daimler, who were investors in the plane maker. Judges said the trial was dropped as they had all been cleared by France’s market regulator AMF, and it would have breached safeguards against double prosecutions

  • 21/50 Alibaba hits back over ‘fake’ claims

    Alibaba has hit out at Gucci’s owner, Kering, for “wasteful litigation instead of the path of constructive co-operation”, after the French luxury giant filed a lawsuit in New York alleging the Chinese online retailer allows sales of fake goods on its websites. Gucci unveiled its Fall-Winter 2015-2016 women’s range at this year’s Milan Fashion Week

    Getty Images

  • 22/50 Hats off to Ralph Lauren

    The upmarket US clothing company Ralph Lauren saw sales climb 1 per cent to $1.9bn (£1.2bn) in the quarter to 28 March. The company said it opened several stores across key markets around the world, fuelling momentum for its luxury accessories business. However, net income decreased to $124m, compared with $153m in the same quarter last year, partly as a result of exchange-rate impacts.

    EPA

  • 23/50 With this ring… arts sales jump

    Sotheby’s raised $380m (£241m) in its New York contemporary art sale, a day after rival Christie’s sold a Picasso for a record $179m, with the auction house describing bidding as a “rush for art”. The top lots included Roy Lichtenstein’s ‘The Ring (Engagement)’ Estate of Roy Lichtenstein

    AP

  • 24/50 Brewdog’s dead cat bounce

    Brewdog parachuted taxidermy “fats cats” holding its latest prospectus out of a helicopter over the City of London as the Scottish brewer looks to drum up publicity for its latest “punk” fundraising. The craft beer group has already raised £5m of the £25m it is seeking from investors to fund a new brewhouse and set up a US operation

  • 25/50 Change at the top for Alibaba

    Chinese ecommerce giant Alibaba replaced its chief executive yesterday. Daniel Zhang, chief operating officer, will replace Jonathan Lu. The move follows the share price briefly dipping below $80 this week, its lowest since its New York flotation in September.

    ChinaFotoPress/ChinaFotoPress via Getty Images

  • 26/50 Put that light out

    New York City officials have been informed by the American Government that the Big Apple’s dazzling neon billboards violate a 2012 highways ruling and must be taken down. The city could forfeit up to $90m (£59m) in federal highway money if it fails to comply. New York is debating getting an exemption from the rule.

    AP

  • 27/50 Lionsgate wants the lion’s share

    Lionsgate yesterday announced plans to finance and co-invest in up to 25 British independent films over the next four years. The Canadian-US entertainment company has previously invested in such UK films as A Little Chaos, starring Kate Winslet

  • 28/50 Tillman planning a ‘new Jaeger’

    The fashion entrepreneur Harold Tillman (centre) has spoken out against the way Jon Moulton’s Better Capital has run Jaeger since taking it over from him and declared he would start a rival business to serve disappointed Jaeger customers. Mr Tillman said turnover at the group had fallen and losses widened since the private equity firm took over.

    2011 Getty Images

  • 29/50 Aer Lingus: end in sight to Irish talks with IAG

    Aer Lingus, the Irish carrier being bid for by British Airways’ owner IAG, yesterday said it is hopeful talks between its suitor and the Irish government will conclude soon. Ireland owns a 25 per cent stake in the airline, which posted a €48m (£35m) first-quarter loss yesterday but said its long-haul business had performed well

  • 30/50 Weir sheds more jobs in oil slump

    Slumping orders for oil and gas projects in the wake of the oil price collapse have forced the engineer Weir to cut another 125 jobs. Since the start of the year, the US oil-rig count has more than halved, sending orders in Weir’s oil and gas division down 23 per cent in the first quarter. Weir has already shed 1,300 staff in the past year

  • 31/50 Williams F1 team drives into the red

    Formula 1 team Williams drove £34m into the red for 2014 after the worst season in its history. A fall in prize money for 2013 – paid a year in arrears – meant the Frankfurt-listed group went into reverse after a £12m profit the year before. But Williams said this year’s figures would be better after it finished third in the 2014 Constructors’ Championship. Its driver Felipe Massa, pictured in Bahrain, is 5th in this year’s driver’s contest

  • 32/50 Rivals roar as Harley stalls

    A strong dollar allowed US motorcycle-maker Harley-Davidson’s foreign competitors to undercut its prices in the first quarter, hurting its sales and market share and prompting it to lower 2015 shipment forecasts. The news sent its shares down 10 per cent in early trading

  • 33/50 Climate changes for BP

    BP will be more accountable for its role in climate change after 98 per cent of investors backed a resolution to make it more transparent on the issue. BP’s annual shareholders’ meeting was held in London yesterday and attracted a protest against offshore oil drilling.

  • 34/50 Car sales lift in Europe

    Demand for new cars in Europe soared last month, particularly in Spain and Portugal where sales increased by more than 40 per cent on March last year. Meanwhile UK sales were up 6.1 per cent on March 2014, according to the European Automobile Manufacturer Association.

  • 35/50 Ricci heir guilt of tax fraud

    Arlette Ricci, the heir to the Nina Ricci perfume and fashion fortune, has been convicted of tax fraud by a Paris court, after hiding millions in an offshore HSBC account. Ms Ricci, 73, was sentenced to a year in prison and ordered to pay a €1m (£72,000) fine.

    Rex

  • 36/50 Camden set for a makeover

    Teddy Sagi, Camden Market’s new Israeli billionaire owner, has appointed Mace as project manager to oversee the redevelopment of Camden Lock Village, with 170 homes as well as shops and offices in the north London district.

    Getty

  • 37/50 All aboard the iron ore train

    Australia is reportedly examining how BHP Billiton and Rio Tinto shift billions of dollars in iron ore profits through low tax hubs in Singapore.

    AFP/Getty Images

  • 38/50 Shell’s Arctic plans under fire

    Greenpeace activists have climbed aboard a Shell oil rig in the Pacific Ocean bound for the Arctic. They plan to camp under the rig’s main deck in protest against the oil giant’s plans to restart drilling off the coast of Alaska. The group said the six protesters would occupy the underside of the Polar Pioneer’s main deck but would not interfere with the vessel.

    Vincenzo Floramo / Greenpeace

  • 39/50 GoDaddy off to flying start in New York

    GoDaddy shares jumped 34 per cent as they started trading on the NYSE Wednesday, valuing the website group at more than $5bn (£3bn). The firm, which manages 59 million web domains – a fifth of the world’s total – raised $460m. It sponsors race car driver Danica Patrick (pictured with chief executive Blake Irving) and is backed by private equity firms KKR and Silver Lake

    EPA/ANDREW GOMBERT

  • 40/50 The first lady of high street banks

    Shriti Vadera, a former adviser to Gordon Brown, has become the first female chairman of a UK high street bank, succeeding Sir Terry Burns at the helm of Santander UK. Baroness Vadera will be paid £650,000 and work alongside Santander’s new CEO, Nathan Bostock

    Paul Morigi/Getty Images

  • 41/50 Sterling returns for Compass

    Compass could see £30m added to its annual profits, thanks to the strength of the pound. The catering giant, which serves up strawberries and cream at Wimbledon each year, said underlying revenues are up 5.5 per cent in the first half, while gains from sterling against the euro, yen and Brazil’s real are set to offset any weakness of the pound against the dollar

    Peter Macdiarmid/Getty Images

  • 42/50 Serco sells Australian rail unit

    Serco has sold its Great Southern Rail to the Australian private equity firm Allegro Funds in a £2.5m deal. The disposal of the luxury business, which runs tourist trains such as The Ghan (pictured) across Australia, is part of the outsourcing group’s move to focus on core businesses under its new chief executive, Rupert Soames

    Mark Metcalfe/Getty Images

  • 43/50 Italian Retailer Yoox eyes Net-a-Porter

    The Italian internet retailer Yoox in the latest contender to swoop in with an offer for the luxury fashion business Net-a-Porter that could value the London-based firm at £1.3 billion

    Splash News/Corbis

  • 44/50 Li Ka-shing buys O2 for £10bn

    Li Ka-shing’s Hutchison Whampoa has finalised a £10.25bn deal to buy the UK mobile business O2 from Spain’s Telefonica. Hutchison, the Hong Kong conglomerate that already owns Three in the UK, entered exclusive talks with Telefonica in January.

    AP

  • 45/50 Jaguar climbs high in the City

    A Hollywood stuntman in a Jaguar XF crosses the River Thames by highwire to promote the latest model. The stunt was supported by 34mm-wide carbon wires.

    PA

  • 46/50 Pubs get protected

    New rules have been announced to mark Community Pubs Day today which protect local pubs and give communities a greater say in planning processes.

    Oli Scarff | Getty Images

  • 47/50 Tesco jet still left on shelf

    Tesco has been forced to cut the price of its Gulfstream G550 jet by £3.5m in an attempt to shift it. The retailer has already sold three of its fleet of five jets since October but has cut the price of the 14-seater jet from $35m (£23.5m) to $30m. It also still has a Hawker 800XP.

    Getty

  • 48/50 Uber called to account in Paris

    Uber’s Paris offices have been raided by French police as part of an investigation into its UberPOP service that lets anyone offer taxi rides. The French headquarters of the US taxi-booking app have been targeted by authorities as part of investigations into the feature, which is not available in the UK, that allows non-professional drivers to sign up

    Reuters

  • 49/50 P&G on the scent of a sale?

    Procter & Gamble is believed to be looking at a sale or IPO of some of its beauty brands, which range from Herbal Essence shampoos to Olay creams and Boss perfumes (as advertised by Sienna Miller, pictured). Shares in the US consumer products giant rose 2 per cent on the prospect of a sale yesterday, even though P&G said the report, on Bloomberg, was “speculation”

  • 50/50 Blackpool pier is on sale for £12.6m

    Blackpool’s central pier has been put up for sale by its owner Cuerden as the attractions organiser looks to restructure its operations. The group is asking for offers of about £12.6m for a package of three piers – including Blackpool’s 19th-century central and southern piers and a third in Landudno in Wales

    Getty

6. Mini bars almost always lose money

Despite the snacks in the minibar seeming like the most overpriced food you have ever seen, hotel owners are still struggling to make a profit from those snacks. "Minibars almost always lose money, even when they charge $10 for a Diet Coke,” Sharon said.

agenda7.jpg
Towels should always be cleaned between stays

7. Always made sure the hand towels are clean when you arrive

Forrest Jones made a discovery when he was helping out with the housekeepers. “You know where you almost always find a hand towel in any recently-vacated hotel room that was occupied by a guy? On the floor, next to the bed, about halfway down, maybe a little toward the foot of the bed. Same spot in the floor, next to almost every bed occupied by a man, in every room. I'll leave the rest to your imagination,” he said.

Comments

    Post
    Share:
    0 Comments
    • Subscribe
    • RSS
    ================================================ FILE: test/test-pages/buzzfeed-1/expected-metadata.json ================================================ { "title": "Student Dies After Diet Pills She Bought Online \"Burned Her Up From Within\"", "byline": null, "dir": null, "lang": "en", "excerpt": "An inquest into Eloise Parry's death has been adjourned until July.", "siteName": "BuzzFeed", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/buzzfeed-1/expected.html ================================================

    The mother of a woman who took suspected diet pills bought online has described how her daughter was “literally burning up from within” moments before her death.

    West Merica Police

    Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a lethal dose of highly toxic “slimming tablets”.

    “The drug was in her system, there was no anti-dote, two tablets was a lethal dose – and she had taken eight,” her mother, Fiona, said in a statement yesterday.

    “As Eloise deteriorated, the staff in A&E did all they could to stabilise her. As the drug kicked in and started to make her metabolism soar, they attempted to cool her down, but they were fighting an uphill battle.

    “She was literally burning up from within.”

    She added: “They never stood a chance of saving her. She burned and crashed.”

    Facebook

    Facebook

    West Mercia police said the tablets were believed to contain dinitrophenol, known as DNP, which is a highly toxic industrial chemical.

    “We are undoubtedly concerned over the origin and sale of these pills and are working with partner agencies to establish where they were bought from and how they were advertised,” said chief inspector Jennifer Mattinson from the West Mercia police.

    The Food Standards Agency warned people to stay away from slimming products that contained DNP.

    “We advise the public not to take any tablets or powders containing DNP, as it is an industrial chemical and not fit for human consumption,” it said in a statement.

    Fiona Parry issued a plea for people to stay away from pills containing the chemical.

    “[Eloise] just never really understood how dangerous the tablets that she took were,” she said. “Most of us don’t believe that a slimming tablet could possibly kill us.

    “DNP is not a miracle slimming pill. It is a deadly toxin.”

    ================================================ FILE: test/test-pages/buzzfeed-1/source.html ================================================ Student Dies After Diet Pills She Bought Online "Burned Her Up From Within" - BuzzFeed News
    News
    Buzz
    Life
    Quizzes
    Videos
    More  
    BuzzFeed Community Make A Post!
    SECTIONS
    Made in NY
    23 choses que toutes les filles des années 2000…
    Sponsorisé
     →
    21 tatouages qui ont merveilleusement bien…
    Voici à quoi ressemblent des vêtements «grande…
    34 raisons de ne jamais aller au Pays basque
    Les 43 fois où Grey’s Anatomy a fait couler toutes…
    23 réactions que toute personne ayant travaillé…
    19 réactions que tous les gens ayant déjà eu…
    Ces femmes postent fièrement des photos de leurs…

    Student Dies After Diet Pills She Bought Online “Burned Her Up From Within”

    An inquest into Eloise Parry’s death has been adjourned until July.

    posted on April 21, 2015, at 11:29 a.m.
    Mark Di Stefano
    Mark Di Stefano
    BuzzFeed News Reporter

    The mother of a woman who took suspected diet pills bought online has described how her daughter was “literally burning up from within” moments before her death.

    The mother of a woman who took suspected diet pills bought online has described how her daughter was "literally burning up from within" moments before her death.

    View this image ›

    West Merica Police

    Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a lethal dose of highly toxic “slimming tablets”.

    “The drug was in her system, there was no anti-dote, two tablets was a lethal dose – and she had taken eight,” her mother, Fiona, said in a statement yesterday.

    “As Eloise deteriorated, the staff in A&E did all they could to stabilise her. As the drug kicked in and started to make her metabolism soar, they attempted to cool her down, but they were fighting an uphill battle.

    “She was literally burning up from within.”

    She added: “They never stood a chance of saving her. She burned and crashed.”

    Facebook

    Facebook

    West Mercia police said the tablets were believed to contain dinitrophenol, known as DNP, which is a highly toxic industrial chemical.

    “We are undoubtedly concerned over the origin and sale of these pills and are working with partner agencies to establish where they were bought from and how they were advertised,” said chief inspector Jennifer Mattinson from the West Mercia police.

    The Food Standards Agency warned people to stay away from slimming products that contained DNP.

    “We advise the public not to take any tablets or powders containing DNP, as it is an industrial chemical and not fit for human consumption,” it said in a statement.

    Fiona Parry issued a plea for people to stay away from pills containing the chemical.


    “[Eloise] just never really understood how dangerous the tablets that she took were,” she said. “Most of us don’t believe that a slimming tablet could possibly kill us.

    “DNP is not a miracle slimming pill. It is a deadly toxin.”

    Check out more articles on BuzzFeed.com!

    Mark di Stefano is a breaking news reporter for BuzzFeed News and is based in Sydney, Australia.
    Contact Mark Di Stefano at mark.distefano@buzzfeed.com
    More ▾
     
     
    Gone Girl: An Interview With An American In ISIS

    by Ellie Hall

    Connect With News

    More News
    More News ›
    Le buzz du moment
    Plus de Buzz ›
      Are you sure you want to remove this item? You can\'t restore it with "Cancel" button!
      NO
      YES
      Please wait...
      This Post Is Locked

      has been editing this post since .

      Uh Oh!

      took your lock at .

      Super Uh Oh!

      Something's wrong, a mini-history of this post:


      What type of post are you making?
      New Article
      List
      Breaking
      ================================================ FILE: test/test-pages/citylab-1/expected-metadata.json ================================================ { "title": "The Modern Ambitions Behind Neon", "byline": "Sarah Archer", "dir": null, "lang": "en", "excerpt": "The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline.", "siteName": "CityLab", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/citylab-1/expected.html ================================================
      The Moulin Rouge cabaret in Paris Benoit Tessier/Reuters

      Why Neon Is the Ultimate Symbol of the 20th Century

      The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline.

      In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s Times Square their perpetual nighttime glow. Using the boiling point of argon as a reference point, Ramsay and his colleague Morris W. Travers isolated three more noble gases and gave them evocative Greek names: neon, krypton, and xenon. In so doing, the scientists bestowed a label of permanent novelty on the most famous of the trio—neon, which translates as “new.” This discovery was the foundation on which the French engineer Georges Claude crafted a new form of illumination over the next decade. He designed glass tubes in which neon gas could be trapped, then electrified, to create a light that glowed reliably for more than 1,000 hours.

      In the 2012 book L’être et le Néon, which has been newly translated into English by Michael Wells, the philosopher Luis de Miranda weaves a history of neon lighting as both artifact and metaphor. Being and Neonness, as the book is called in its English edition, isn’t a typical material history. There are no photographs. Even de Miranda’s own example of a neon deli sign spotted in Paris is re-created typographically, with text in all caps and dashes forming the border of the sign, as one might attempt on Twitter. Fans of Miami Beach’s restored Art Deco hotels and California’s bowling alleys might be disappointed by the lack of glossy historical images. Nonetheless, de Miranda makes a convincing case for neon as a symbol of the grand modern ambitions of the 20th century.

      De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in his introduction: “When we hear the word neon, an image pops into our heads: a combination of light, colors, symbols, and glass. This image is itself a mood. It carries an atmosphere. It speaks … of the essence of cities, of the poetry of nights, of the 20th century.” When neon lights debuted in Europe, they seemed dazzlingly futuristic. But their husky physicality started becoming obsolete by the 1960s, thanks in part to the widespread use of plastic for fluorescent signs. Neon signs exist today, though they’ve been eclipsed by newer technologies such as digital billboards, and they remain charmingly analog: Signs must be made by hand because there’s no cost-effective way to mass-produce them.

      In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely the time and place where the first great modernist works were being created. De Miranda’s recounting of the ingenuity emerging from the French capital a century ago is thrilling to contemplate: the cubist art of Pablo Picasso, the radically deconstructed fashions of Coco Chanel, the stream-of-consciousness poetry of Gertrude Stein, and the genre-defying music of Claude Debussy—all of which heralded a new age of culture for Europe and for the world.

      Amid this artistic groundswell, Georges Claude premiered his neon lights at the Paris Motor Show in December 1910, captivating visitors with 40-foot-tall tubes affixed to the building’s exterior. The lights shone orange-red because neon, by itself, produces that color. Neon lighting is a catchall term that describes the technology of glass tubing that contains gas or chemicals that glow when electrified. For example, neon fabricators use carbon dioxide to make white, and mercury to make blue. Claude acknowledged at the time that neon didn’t produce the ideal color for a standard light bulb and insisted that it posed no commercial threat to incandescent bulbs.

      Of course, the very quality that made neon fixtures a poor choice for interior lighting made them perfect for signs, de Miranda notes. The first of the neon signs was switched on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre, and eventually they were adopted by cinemas and nightclubs. While Claude had a monopoly on neon lighting throughout the 1920s, the leaking of trade secrets and the expiration of a series of patents broke his hold on the rapidly expanding technology.

      In the following decades, neon’s nonstop glow and vibrant colors turned ordinary buildings and surfaces into 24/7 billboards for businesses, large and small, that wanted to convey a sense of always being open. The first examples of neon in the United States debuted in Los Angeles, where the Packard Motor Car Company commissioned two large blue-and-orange Packard signs that literally stopped traffic because they distracted motorists. The lighting also featured heavily at the Chicago Century of Progress Exposition in 1933 and at the 1939 World’s Fair in New York. At the latter event, a massive neon sign reading Futurama lit the way to a General Motors exhibition that heralded “The World of Tomorrow.”

      Workers remove a hammer and sickle from a neon sign that reads “Glory to Communism,” visible on the roof of the Communist-run electricity-board headquarters in Czechoslovakia in 1989. (AP)

      De Miranda points out that businesses weren’t alone in embracing neon’s ability to spread messages effectively. By the middle of the century, the lighting was being adopted for more political purposes. “In the 1960s, the Soviets deployed a vast ‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw], they did not hesitate to illuminate the façades of public buildings.” In other words, as opposed to the sole use of the more obvious forms of propaganda via posters or slogans, the mass introduction of neon lighting was a way of getting citizens of Communist cities to see their surroundings with the pizzazz and nighttime glamour of major Western capitals.

      Neon, around this time, began to be phased out, thanks to cheaper and less labor-intensive alternatives. In addition, the global economic downturn of the 1970s yielded a landscape in which older, flickering neon signs, which perhaps their owners couldn’t afford to fix or replace, came to look like symbols of decline. Where such signs were once sophisticated and novel, they now seemed dated and even seedy.

      Cities are changing fast. Keep up with the CityLab Daily newsletter.

      De Miranda understands this evolution by zooming out and looking at the 1900s as the “neon century.” The author draws a parallel between the physical form of neon lights, which again are essentially containers for electrified gases, and that of a glass capsule—suggesting they are a kind of message in a bottle from a time before the First World War. “Since then, [neon lights] have witnessed all the transformations that have created the world we live in,” de Miranda writes. “Today, they sometimes seem to maintain a hybrid status, somewhere between junkyards and museums, not unlike European capitals themselves.”

      Martin Wartman, a student at Northern Kentucky University, works on a neon sign at the Neonworks of Cincinnati workshop connected to the American Sign Museum, in 2016. (John Minchillo / AP)

      Another mark of neon’s hybridity: Its obsolescence started just as some contemporary artists began using the lights in their sculptures. Bruce Nauman’s 1968 work My Name as Though It Were Written on the Surface of the Moon poked fun at the space race—another symbol of 20th-century technological innovation whose moment has passed. The piece uses blue “neon” letters (mercury, actually) to spell out the name “bruce” in lowercase cursive, with each character repeated several times as if to convey a person speaking slowly in outer space. The British artist Tracey Emin has made sculptures that resemble neon Valentine’s Day candies: They read as garish and sentimental confections with pink, heart-shaped frames that surround blue text fragments. Drawing on the nostalgia-inducing quality of neon, the sculptures’ messages are redolent of old-fashioned movie dialogue, with titles such as “You Loved Me Like a Distant Star” and “The Kiss Was Beautiful.”

      Seeing neon lighting tamed in the context of a gallery display fits comfortably with de Miranda’s notion that neon technology is like a time capsule from another age. In museums, works of neon art and design coexist with objects that were ahead of their own time in years past—a poignant fate for a technology that made its name advertising “The World of Tomorrow.” Yet today neon is also experiencing a kind of craft revival. The fact that it can’t be mass-produced has made its fabrication something akin to a cherished artisanal technique. Bars and restaurants hire firms such as Let There Be Neon in Manhattan, or the L.A.-based master neon artist Lisa Schulte, to create custom signs and works of art. Neon’s story even continues to glow from inside museums such as California’s Museum of Neon Art and the Neon Museum in Las Vegas. If it can still be a vital medium for artists and designers working today, “neonness” need not only be trapped in the past. It might also capture the mysterious glow of the near future—just as it did a century ago.

      This article originally appeared on The Atlantic.

      About the Author

      ================================================ FILE: test/test-pages/citylab-1/source.html ================================================ A Brief History of Neon Signage - CityLab
      The Moulin Rouge cabaret in Paris Benoit Tessier/Reuters

      Why Neon Is the Ultimate Symbol of the 20th Century

      1. Sarah Archer

      The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline.

      In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s Times Square their perpetual nighttime glow. Using the boiling point of argon as a reference point, Ramsay and his colleague Morris W. Travers isolated three more noble gases and gave them evocative Greek names: neon, krypton, and xenon. In so doing, the scientists bestowed a label of permanent novelty on the most famous of the trio—neon, which translates as “new.” This discovery was the foundation on which the French engineer Georges Claude crafted a new form of illumination over the next decade. He designed glass tubes in which neon gas could be trapped, then electrified, to create a light that glowed reliably for more than 1,000 hours.

      In the 2012 book L’être et le Néon, which has been newly translated into English by Michael Wells, the philosopher Luis de Miranda weaves a history of neon lighting as both artifact and metaphor. Being and Neonness, as the book is called in its English edition, isn’t a typical material history. There are no photographs. Even de Miranda’s own example of a neon deli sign spotted in Paris is re-created typographically, with text in all caps and dashes forming the border of the sign, as one might attempt on Twitter. Fans of Miami Beach’s restored Art Deco hotels and California’s bowling alleys might be disappointed by the lack of glossy historical images. Nonetheless, de Miranda makes a convincing case for neon as a symbol of the grand modern ambitions of the 20th century.

      De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in his introduction: “When we hear the word neon, an image pops into our heads: a combination of light, colors, symbols, and glass. This image is itself a mood. It carries an atmosphere. It speaks … of the essence of cities, of the poetry of nights, of the 20th century.” When neon lights debuted in Europe, they seemed dazzlingly futuristic. But their husky physicality started becoming obsolete by the 1960s, thanks in part to the widespread use of plastic for fluorescent signs. Neon signs exist today, though they’ve been eclipsed by newer technologies such as digital billboards, and they remain charmingly analog: Signs must be made by hand because there’s no cost-effective way to mass-produce them.

      In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely the time and place where the first great modernist works were being created. De Miranda’s recounting of the ingenuity emerging from the French capital a century ago is thrilling to contemplate: the cubist art of Pablo Picasso, the radically deconstructed fashions of Coco Chanel, the stream-of-consciousness poetry of Gertrude Stein, and the genre-defying music of Claude Debussy—all of which heralded a new age of culture for Europe and for the world.

      Amid this artistic groundswell, Georges Claude premiered his neon lights at the Paris Motor Show in December 1910, captivating visitors with 40-foot-tall tubes affixed to the building’s exterior. The lights shone orange-red because neon, by itself, produces that color. Neon lighting is a catchall term that describes the technology of glass tubing that contains gas or chemicals that glow when electrified. For example, neon fabricators use carbon dioxide to make white, and mercury to make blue. Claude acknowledged at the time that neon didn’t produce the ideal color for a standard light bulb and insisted that it posed no commercial threat to incandescent bulbs.

      Of course, the very quality that made neon fixtures a poor choice for interior lighting made them perfect for signs, de Miranda notes. The first of the neon signs was switched on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre, and eventually they were adopted by cinemas and nightclubs. While Claude had a monopoly on neon lighting throughout the 1920s, the leaking of trade secrets and the expiration of a series of patents broke his hold on the rapidly expanding technology.

      In the following decades, neon’s nonstop glow and vibrant colors turned ordinary buildings and surfaces into 24/7 billboards for businesses, large and small, that wanted to convey a sense of always being open. The first examples of neon in the United States debuted in Los Angeles, where the Packard Motor Car Company commissioned two large blue-and-orange Packard signs that literally stopped traffic because they distracted motorists. The lighting also featured heavily at the Chicago Century of Progress Exposition in 1933 and at the 1939 World’s Fair in New York. At the latter event, a massive neon sign reading Futurama lit the way to a General Motors exhibition that heralded “The World of Tomorrow.”

      Workers remove a hammer and sickle from a neon sign that reads “Glory to Communism,” visible on the roof of the Communist-run electricity-board headquarters in Czechoslovakia in 1989. (AP)

      De Miranda points out that businesses weren’t alone in embracing neon’s ability to spread messages effectively. By the middle of the century, the lighting was being adopted for more political purposes. “In the 1960s, the Soviets deployed a vast ‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw], they did not hesitate to illuminate the façades of public buildings.” In other words, as opposed to the sole use of the more obvious forms of propaganda via posters or slogans, the mass introduction of neon lighting was a way of getting citizens of Communist cities to see their surroundings with the pizzazz and nighttime glamour of major Western capitals.

      Neon, around this time, began to be phased out, thanks to cheaper and less labor-intensive alternatives. In addition, the global economic downturn of the 1970s yielded a landscape in which older, flickering neon signs, which perhaps their owners couldn’t afford to fix or replace, came to look like symbols of decline. Where such signs were once sophisticated and novel, they now seemed dated and even seedy.

      De Miranda understands this evolution by zooming out and looking at the 1900s as the “neon century.” The author draws a parallel between the physical form of neon lights, which again are essentially containers for electrified gases, and that of a glass capsule—suggesting they are a kind of message in a bottle from a time before the First World War. “Since then, [neon lights] have witnessed all the transformations that have created the world we live in,” de Miranda writes. “Today, they sometimes seem to maintain a hybrid status, somewhere between junkyards and museums, not unlike European capitals themselves.”

      Martin Wartman, a student at Northern Kentucky University, works on a neon sign at the Neonworks of Cincinnati workshop connected to the American Sign Museum, in 2016. (John Minchillo / AP)

      Another mark of neon’s hybridity: Its obsolescence started just as some contemporary artists began using the lights in their sculptures. Bruce Nauman’s 1968 work My Name as Though It Were Written on the Surface of the Moon poked fun at the space race—another symbol of 20th-century technological innovation whose moment has passed. The piece uses blue “neon” letters (mercury, actually) to spell out the name “bruce” in lowercase cursive, with each character repeated several times as if to convey a person speaking slowly in outer space. The British artist Tracey Emin has made sculptures that resemble neon Valentine’s Day candies: They read as garish and sentimental confections with pink, heart-shaped frames that surround blue text fragments. Drawing on the nostalgia-inducing quality of neon, the sculptures’ messages are redolent of old-fashioned movie dialogue, with titles such as “You Loved Me Like a Distant Star” and “The Kiss Was Beautiful.”

      Seeing neon lighting tamed in the context of a gallery display fits comfortably with de Miranda’s notion that neon technology is like a time capsule from another age. In museums, works of neon art and design coexist with objects that were ahead of their own time in years past—a poignant fate for a technology that made its name advertising “The World of Tomorrow.” Yet today neon is also experiencing a kind of craft revival. The fact that it can’t be mass-produced has made its fabrication something akin to a cherished artisanal technique. Bars and restaurants hire firms such as Let There Be Neon in Manhattan, or the L.A.-based master neon artist Lisa Schulte, to create custom signs and works of art. Neon’s story even continues to glow from inside museums such as California’s Museum of Neon Art and the Neon Museum in Las Vegas. If it can still be a vital medium for artists and designers working today, “neonness” need not only be trapped in the past. It might also capture the mysterious glow of the near future—just as it did a century ago.

      This article originally appeared on The Atlantic.

      About the Author

      Most Popular

      1. A photo of an Uber Bus in Cairo, the company's new microtransit program.
        Transportation

        ‘Uber Was Supposed To Be Our Public Transit’

        Innisfil, Ontario, decided to partially subsidize ride-hailing trips rather than pay for a public bus system. It worked so well that now they have to raise fares and cap rides.

        1. Laura Bliss
      2. Environment

        A 13,235-Mile Road Trip for 70-Degree Weather Every Day

        This year-long journey across the U.S. keeps you at consistent high temperatures.

        1. John Metcalfe
      3. The outside of a large apartment building, dotted with air conditioners.
        Environment

        What If Air Conditioning Could Help Stop Climate Change Instead of Causing It?

        Using technology currently in development, AC units in skyscrapers and homes could get turned into machines that pull carbon dioxide out of the atmosphere.

        1. Matt Simon
      4. A train traveling on an elevated platform.
        Transportation

        Texas High Speed Rail Faces a New Threat: Semantics

        A private company plans to break ground on a bullet train between Houston and Dallas in 2019. But opponents of the project have a new argument.

        1. Laura Bliss
      5. Solutions

        In Switzerland, Everyone’s an Urban Planner

        To reimagine its largest public space, the Swiss city of Lausanne organized a citywide consultation and workshop that asked: Just who is the public?

        1. Feargus O'Sullivan
      Maps
      Click Here
      Newsletters
      Click Here
      Facebook
      Click Here
      Skip to content
      Popular Now on CityLab »
      ================================================ FILE: test/test-pages/clean-links/expected-metadata.json ================================================ { "title": "Bartleby the Scrivener Web Study Text", "byline": null, "dir": null, "excerpt": "Ere introducing the scrivener, as he first appeared to me, it is fit \n I make some mention of myself, my employees, my business, my chambers, \n and general surroundings; because some such description is indispensable \n to an adequate understanding of the chief character about to be presented.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/clean-links/expected.html ================================================

      Study Webtext

      "Bartleby the Scrivener: A Story of Wall-Street " (1853) 
      Herman Melville

      To the story text without notes

      Prepared by Ann Woodlief, Virginia Commonwealth University

      Click on text in red for hypertext notes and questions
      I am a rather elderly man. The nature of my avocations for the last thirty years has brought me into more than ordinary contact with what would seem an interesting and somewhat singular set of men of whom as yet nothing that I know of has ever been written:-- I mean the law-copyists or scriveners. I have known very many of them, professionally and privately, and if I pleased, could relate divers histories, at which good-natured gentlemen might smile, and sentimental souls might weep. But I waive the biographies of all other scriveners for a few passages in the life of Bartleby, who was a scrivener the strangest I ever saw or heard of. While of other law-copyists I might write the complete life, of Bartleby nothing of that sort can be done. I believe that no materials exist for a full and satisfactory biography of this man. It is an irreparable loss to literature. Bartleby was one of those beings of whom nothing is ascertainable, except from the original sources, and in his case those are very small. What my own astonished eyes saw of Bartleby, that is all I know of him, except, indeed, one vague report which will appear in the sequel.

      Ere introducing the scrivener, as he first appeared to me, it is fit I make some mention of myself, my employees, my business, my chambers, and general surroundings; because some such description is indispensable to an adequate understanding of the chief character about to be presented.

      Imprimis: I am a man who, from his youth upwards, has been filled with a profound conviction that the easiest way of life is the best.. Hence, though I belong to a profession proverbially energetic and nervous, even to turbulence, at times, yet nothing of that sort have I ever suffered to invade my peace. I am one of those unambitious lawyers who never addresses a jury, or in any way draws down public applause; but in the cool tranquillity of a snug retreat, do a snug business among rich men's bonds and mortgages and title-deeds. The late John Jacob Astor, a personage little given to poetic enthusiasm, had no hesitation in pronouncing my first grand point to be prudence; my next, method. I do not speak it in vanity, but simply record the fact, that I was not unemployed in my profession by the last John Jacob Astor; a name which, I admit, I love to repeat, for it hath a rounded and orbicular sound to it, and rings like unto bullion. I will freely add, that I was not insensible to the late John Jacob Astor's good opinion.

      Some time prior to the period at which this little history begins, my avocations had been largely increased. The good old office, now extinct in the State of New York, of a Master in Chancery, had been conferred upon me. It was not a very arduous office, but very pleasantly remunerative. I seldom lose my temper; much more seldom indulge in dangerous indignation at wrongs and outrages; but I must be permitted to be rash here and declare, that I consider the sudden and violent abrogation of the office of Master of Chancery, by the new Constitution, as a----premature act; inasmuch as I had counted upon a life-lease of the profits, whereas I only received those of a few short years. But this is by the way.

      My chambers were up stairs at No.--Wall-street. At one end they looked upon the white wall of the interior of a spacious sky-light shaft, penetrating the building from top to bottom. This view might have been considered rather tame than otherwise, deficient in what landscape painters call "life." But if so, the view from the other end of my chambers offered, at least, a contrast, if nothing more. In that direction my windows commanded an unobstructed view of a lofty brick wall,black by age and everlasting shade; which wall required no spy-glass to bring out its lurking beauties, but for the benefit of all near-sighted spectators, was pushed up to within ten feet of my window panes. Owing to the great height of the surrounding buildings, and my chambers being on the second floor, the interval between this wall and mine not a little resembled a huge square cistern.

      At the period just preceding the advent of Bartleby, I had two persons as copyists in my employment, and a promising lad as an office-boy. First, Turkey; second, Nippers; third, Ginger Nut.These may seem names, the like of which are not usually found in the Directory. In truth they were nicknames, mutually conferred upon each other by my three clerks, and were deemed expressive of their respective persons or characters. Turkey was a short, pursy Englishman of about my own age, that is, somewhere not far from sixty. In the morning, one might say, his face was of a fine florid hue, but after twelve o'clock, meridian-- his dinner hour-- it blazed like a grate full of Christmas coals; and continued blazing--but, as it were, with a gradual wane--till 6 o'clock, P.M. or thereabouts, after which I saw no more of the proprietor of the face, which gaining its meridian with the sun, seemed to set with it, to rise, culminate, and decline the following day, with the like regularity and undiminished glory. There are many singular coincidences I have known in the course of my life, not the least among which was the fact that exactly when Turkey displayed his fullest beams from his red and radiant countenance, just then, too, at the critical moment, began the daily period when I considered his business capacities as seriously disturbed for the remainder of the twenty-four hours. Not that he was absolutely idle, or averse to business then; far from it. The difficulty was, he was apt to be altogether too energetic. There was a strange, inflamed, flurried, flighty recklessness of activity about him. He would be incautious in dipping his pen into his inkstand. All his blots upon my documents, were dropped there after twelve o'clock, meridian. Indeed, not only would he be reckless and sadly given to making blots in the afternoon, but some days he went further, and was rather noisy. At such times, too, his face flamed with augmented blazonry, as if cannel coal had been heaped on anthracite. He made an unpleasant racket with his chair; spilled his sand-box; in mending his pens, impatiently split them all to pieces, and threw them on the floor in a sudden passion; stood up and leaned over his table, boxing his papers about in a most indecorous manner, very sad to behold in an elderly manlike him. Nevertheless, as he was in many ways a most valuable person to me, and all the time before twelve o'clock, meridian, was the quickest, steadiest creature too, accomplishing a great deal of work in a style not easy to be matched--for these reasons, I was willingto overlook his eccentricities, though indeed, occasionally, I remonstrated with him. I did this very gently, however, because, though the civilest, nay, the blandest and most reverential of men in the morning, yet in the afternoon he was disposed, upon provocation, to be slightly rash with his tongue, in fact, insolent. Now, valuing his morning services as I did, and resolved not to lose them; yet, at the same time made uncomfortable by his inflamed ways after twelve o'clock; and being a man of peace, unwilling by my admonitions to call forth unseemingly retorts from him; I took upon me, one Saturday noon (he was always worse on Saturdays), to hint to him, very kindly, that perhaps now that he was growing old, it might be well to abridge his labors; in short, he need not come to my chambers after twelve o'clock, but, dinner over, had best go home to his lodgings and rest himself till tea-time. But no; he insisted upon his afternoon devotions. His countenance became intolerably fervid, as he oratorically assured me--gesticulating with a long ruler at the other end of the room--that if his services in the morning were useful, how indispensible, then, in the afternoon?

      "With submission, sir," said Turkey on this occasion, "I consider myself your right-hand man. In the morning I but marshal and deploy my columns; but in the afternoon I put myself at their head, and gallantly charge the foe, thus!"--and he made a violent thrust with the ruler.

      "But the blots, Turkey," intimated I.

      "True,--but, with submission, sir, behold these hairs! I am getting old. Surely, sir, a blot or two of a warm afternoon is not the page--is honorable. With submission, sir, we both are getting old."

      This appeal to my fellow-feeling was hardly to be resisted. At all events, I saw that go he would not. So I made up my mind to let him stay, resolving, nevertheless, to see to it, that during the afternoon he had to do with my less important papers.

      Nippers, the second on my list, was a whiskered, sallow, and, upon the whole, rather piratical-looking young man of about five and twenty. I always deemed him the victim of two evil powers-- ambition and indigestion. The ambition was evinced by a certain impatience of the duties of a mere copyist, an unwarrantable usurpation of strictly profession affairs, such as the original drawing up of legal documents. The indigestion seemed betokened in an occasional nervous testiness and grinning irritability, causing the teeth to audibly grind together over mistakes committed in copying; unnecessary maledictions, hissed, rather than spoken, in the heat of business; and especially by a continual discontent with the height of the table where he worked. Though of a very ingenious mechanical turn, Nippers could never get this table to suit him. He put chips under it, blocks of various sorts, bits of pasteboard, and at last went so far as to attempt an exquisite adjustment by final pieces of folded blotting-paper. But no invention would answer. If, for the sake of easing his back, he brought the table lid at a sharp angle well up towards his chin, and wrote there like a man using the steep roof of a Dutch house for his desk:--then he declared that it stopped the circulation in his arms. If now he lowered the table to his waistbands, and stooped over it in writing, then there was a sore aching in his back. In short, the truth of the matter was, Nippers knew not what he wanted. Or, if he wanted anything, it was to be rid of a scrivener's table altogether. Among the manifestations of his diseased ambition was a fondness he had for receiving visits from certain ambiguous-looking fellows in seedy coats, whom he called his clients. Indeed I was aware that not only was he, at times, considerable of a ward-politician, but he occasionally did a little businessat the Justices' courts, and was not unknown on the steps of the Tombs. I have good reason to believe, however, that one individual who called upon him at my chambers, and who, with a grand air, he insisted was his client, was no other than a dun, and the alleged title-deed, a bill. But with all his failings, and the annoyances he caused me, Nippers, like his compatriot Turkey, was a very useful man to me; wrote a neat, swift hand; and, when he chose, was not deficient in a gentlemanly sort of deportment. Added to this, he always dressedin a gentlemanly sort of way; and so, incidentally, reflected credit upon my chambers. Whereas with respect to Turkey, I had much ado to keep him from being a reproach to me. His clothes were apt to look oily and smell of eating-houses. He wore his pantaloons very loose and baggy in summer. His coats were execrable; his hat not to be handled. But while the hat was a thing of indifference to me, inasmuch as his natural civility and deference, as a dependent Englishman, always led him to doff it the moment he entered the room, yet his coat was another matter. Concerning his coats, I reasoned with him; but with no effect. The truth was, I suppose, that a man with so small an income, could not afford to sport such a lustrous face and a lustrous coat at one and the same time. As Nippers once observed, Turkey's money went chiefly for red ink. One winter day I presented Turkey with a highly-respectable looking coat of my own, a padded gray coat, of a most comfortable warmth, and which buttoned straight up from the knee to the neck. I thought Turkey would appreciate the favor, and abate his rashness and obstreperousness of afternoons. But no. I verily believe that buttoning himself up in so downy and blanket-like a coat had a pernicious effect upon him; upon the same principle that too much oats are bad for horses. In fact, precisely as a rash, restive horse is said to feel his oats, so Turkey felt his coat. It made him insolent. He was a man whom prosperity harmed.

      Though concerning the self-indulgent habits of Turkey I had my own private surmises, yet touching Nippers I was well persuaded that whatever might be his faults in other respects, he was, at least, a temperate young man. But indeed, nature herself seemed to have been his vintner, and at his birth charged him so thoroughly with an irritable, brandy-like disposition, that all subsequent potations were needless. When I consider how, amid the stillness of my chambers, Nippers would sometimes impatiently rise from his seat, and stooping over his table, spread his arms wide apart, seize the whole desk, and move it, and jerk it, with a grim, grinding motion on the floor, as if the table were a perverse voluntary agent, intent on thwarting and vexing him; I plainly perceive that for Nippers, brandy and water were altogether superfluous.

      It was fortunate for me that, owing to its course--indigestion--the irritability and consequent nervousness of Nippers, were mainly observable in the morning, while in the afternoon he was comparatively mild. So that Turkey's paroxysms only coming on about twelve o'clock, I never had to do with their eccentricities at one time. Their fits relieved each other like guards. When Nippers' was on, Turkey's was off, and vice versa. This was a good natural arrangement under the circumstances.

      Ginger Nut, the third on my list, was a lad some twelve years old. His father was a carman, ambitious of seeing his son on the bench instead of a cart, before he died. So he sent him to my office as a student at law, errand boy, and cleaner and sweeper, at the rate of one dollar a week. He had a little desk to himself, but he did not use it much. Upon inspection, the drawer exhibited a great array of the shells of various sorts of nuts. Indeed, to this quick-witted youth the whole noble science of the law was contained in a nut-shell. Not the least among the employments of Ginger Nut, as well as one which he discharged with the most alacrity, was his duty as cake and apple purveyor for Turkey and Nippers. Copying law papers being proverbially a dry, husky sort of business, my two scriveners were fain to moisten their mouths very often with Spitzenbergs to be had at the numerous stalls nigh the Custom House and Post Office. Also, they sent Ginger Nut very frequently for that peculiar cake--small, flat, round, and very spicy--after which he had been named by them. Of a cold morning when business was but dull, Turkey would gobble up scores of these cakes, as if they were mere wafers--indeed they sell them at the rate of six or eight for a penny--the scrape of his pen blending with the crunching of the crisp particles in his mouth. Of all the fiery afternoon blunders and flurried rashnesses of Turkey, was his once moistening a ginger-cake between his lips, and clapping it on to a mortgage for a seal. I came within an ace of dismissing him then. But he mollified me by making an oriental bow, and saying--"With submission, sir, it was generous of me to find you in stationery on my own account."

      Now my original business--that of a conveyancer and title hunter, and drawer-up of recondite documents of all sorts--was considerably increased by receiving the master's office. There was now great work for scriveners. Not only must I push the clerks already with me, but I must have additional help. In answer to my advertisement, a motionless young man one morning, stood upon my office threshold, the door being open, for it was summer. I can see that figure now--pallidly neat, pitiably respectable, incurably forlorn! It was Bartleby.

      After a few words touching his qualifications, I engaged him, glad to have among my corps of copyists a man of so singularly sedate an aspect, which I thought might operate beneficially upon the flighty temper of Turkey, and the fiery one of Nippers.

      I should have stated before that ground glass folding-doors divided my premises into two parts, one of which was occupied by my scriveners, the other by myself. According to my humor I threw open these doors, or closed them. I resolved to assign Bartleby a corner by the folding-doors, but on my side of them, so as to have this quiet man within easy call, in case any trifling thing was to be done. I placed his desk close up to a small side window in that part of the room, a window which originally had afforded a lateral view of certain grimy back-yards and bricks, but which, owing to subsequent erections, commanded at present no view at all, though it gave some light. Within three feet of the panes was a wall, and the light came down from far above, between two lofty buildings, as from a very small opening in a dome. Still further to a satisfactory arrangement, I procured a high green folding screen, which might entirely isolate Bartleby from my sight, though not remove him from my voice. And thus, in a manner, privacy and society were conjoined.

      At first Bartleby did an extraordinary quantity of writing. As if long famishingfor something to copy, he seemed to gorge himself on my documents. There was no pause for digestion. He ran a day and night line, copying by sun-light and by candle-light. I should have been quite delighted with his application, had be been cheerfully industrious. But he wrote on silently, palely, mechanically.

      It is, of course, an indispensable part of a scrivener's business to verify the accuracy of his copy, word by word. Where there are two or more scriveners in an office, they assist each other in this examination, one reading from the copy, the other holding the original. It is a very dull, wearisome, and lethargic affair. I can readily imagine that to some sanguine temperaments it would be altogether intolerable. For example, I cannot credit that the mettlesome poet Byron would have contentedly sat down with Bartleby to examine a law document of, say five hundred pages, closely written in a crimpy hand.

      Now and then, in the haste of business, it had been my habit to assist in comparing some brief document myself, calling Turkey or Nippers for this purpose. One object I had in placing Bartleby so handy to me behind the screen, was to avail myself of his services on such trivial occasions. It was on the third day, I think, of his being with me, and before any necessity had arisen for having his own writing examined, that, being much hurried to complete a small affair I had in hand, I abruptly called to Bartleby. In my haste and natural expectancy of instant compliance, I sat with my head bent over the original on my desk, and my right hand sideways, and somewhat nervously extended with the copy, so that immediately upon emerging from his retreat, Bartleby might snatch it and proceed to business without the least delay.

      In this very attitude did I sit when I called to him, rapidly stating what it was I wanted him to do--namely, to examine a small paper with me. Imagine my surprise, nay, my consternation, when without moving from his privacy, Bartleby in a singularly mild, firm voice, replied,"I would prefer not to."

      I sat awhile in perfect silence, rallying my stunned faculties. Immediately it occurred to me that my ears had deceived me, or Bartleby had entirely misunderstood my meaning. I repeated my request in the clearest tone I could assume. But in quite as clear a one came the previous reply, "I would prefer not to."

      "Prefer not to," echoed I, rising in high excitement, and crossing the room with a stride, "What do you mean? Are you moon-struck? I want you to help me compare this sheet here--take it," and I thrust it towards him.

      "I would prefer not to," said he.

      I looked at him steadfastly. His face was leanly composed; his gray eye dimly calm. Not a wrinkle of agitation rippled him. Had there been the least uneasiness, anger, impatience or impertinence in his manner; in other words, had there been any thing ordinarily human about him, doubtless I should have violently dismissed him from the premises. But as it was, I should have as soon thought of turning my pale plaster-of-paris bust of Cicero out of doors. I stood gazing at him awhile, as he went on with his own writing, and then reseated myself at my desk. This is very strange, thought I. What had one best do? But my business hurried me. I concluded to forget the matter for the present, reserving it for my future leisure. So calling Nippers from the other room, the paper was speedily examined.

      A few days after this, Bartleby concluded four lengthy documents, being quadruplicates of a week's testimony taken before me in my High Court of Chancery. It became necessary to examine them. It was an important suit, and great accuracy was imperative. Having all things arranged I called Turkey, Nippers and Ginger Nut from the next room, meaning to place the four copies in the hands of my four clerks, while I should read from the original. Accordingly Turkey, Nippers and Ginger Nut had taken their seats in a row, each with his document in hand, when I called to Bartleby to join this interesting group.

      "Bartleby! quick, I am waiting."

      I heard a low scrape of his chair legs on the unscraped floor, and soon he appeared standing at the entrance of his hermitage.

      "What is wanted?" said he mildly.

      "The copies, the copies," said I hurriedly. "We are going to examine them. There"--and I held towards him the fourth quadruplicate.

      "I would prefer not to," he said, and gently disappeared behind the screen.

      For a few moments I was turned into a pillar of salt, standing at the head of my seated column of clerks. Recovering myself, I advanced towards the screen, and demanded the reason for such extraordinary conduct.

      "Why do you refuse?"

      "I would prefer not to."

      With any other man I should have flown outright into a dreadful passion, scorned all further words, and thrust him ignominiously from my presence. But there was something about Bartleby that not only strangely disarmed me, but in a wonderful manner touched and disconcerted me. I began to reason with him.

      "These are your own copies we are about to examine. It is labor saving to you, because one examination will answer for your four papers. It is common usage. Every copyist is bound to help examine his copy. Is it not so? Will you not speak? Answer!"

      "I prefer not to," he replied in a flute-like tone. It seemed to me that while I had been addressing him, he carefully revolved every statement that I made; fully comprehended the meaning; could not gainsay the irresistible conclusion; but, at the same time, some paramount consideration prevailed with him to reply as he did.

      "You are decided, then, not to comply with my request--a request made according to common usage and common sense?"

      He briefly gave me to understand that on that point my judgment was sound. Yes: his decision was irreversible.

      It is not seldom the case that when a man is browbeaten in some unprecedented and violently unreasonable way, he begins to stagger in his own plainest faith. He begins, as it were, vaguely to surmise that, wonderful as it may be, all the justice and all the reason is on the other side. Accordingly, if any disinterested persons are present, he turns to them for some reinforcement for his own faltering mind.

      "Turkey," said I, "what do you think of this? Am I not right?"

      "With submission, sir," said Turkey, with his blandest tone, "I think that you are."

      "Nippers," said I, "what do you think of it?"

      "I think I should kick him out of the office."

      (The reader of nice perceptions will here perceive that, it being morning, Turkey's answer is couched in polite and tranquil terms, but Nippers replies in ill-tempered ones. Or, to repeat a previous sentence, Nipper's ugly mood was on duty, and Turkey's off.)

      "Ginger Nut," said I, willing to enlist the smallest suffrage in my behalf, "what do you think of it?"

      "I think, sir, he's a little luny," replied Ginger Nut, with a grin.

      "You hear what they say," said I, turning towards the screen, "come forth and do your duty."

      But he vouchsafed no reply. I pondered a moment in sore perplexity. But once more business hurried me. I determined again to postpone the consideration of this dilemma to my future leisure. With a little trouble we made out to examine the papers without Bartleby, though at every page or two, Turkey deferentially dropped his opinion that this proceeding was quite out of the common; while Nippers, twitching in his chair with a dyspeptic nervousness, ground out between his set teeth occasional hissing maledictions against the stubborn oaf behind the screen. And for his (Nipper's) part, this was the first and the last time he would do another man's business without pay.

      Meanwhile Bartleby sat in his hermitage, oblivious to every thing but his own peculiar business there.

      Some days passed, the scrivener being employed upon another lengthy work. His late remarkable conduct led me to regard his way narrowly. I observed that he never went to dinner; indeed that he never went any where. As yet I had never of my personal knowledge known him to be outside of my office. He was a perpetual sentry in the corner. At about eleven o'clock though, in the morning, I noticed that Ginger Nut would advance toward the opening in Bartleby's screen, as if silently beckoned thither by a gesture invisible to me where I sat. That boy would then leave the office jingling a few pence, and reappear with a handful of ginger-nuts which he delivered in the hermitage, receiving two of the cakes for his trouble.

      He lives, then, on ginger-nuts, thought I; never eats a dinner, properly speaking; he must be a vegetarian then, but no; he never eats even vegetables, he eats nothing but ginger-nuts. My mind then ran on in reveries concerning the probable effects upon the human constitution of living entirely on ginger-nuts. Ginger-nuts are so called because they contain ginger as one of their peculiar constituents, and the final flavoring one. Now what was ginger? A hot, spicy thing. Was Bartleby hot and spicy? Not at all. Ginger, then, had no effect upon Bartleby. Probably he preferred it should have none.

      Nothing so aggravates an earnest person as a passive resistance. If the individual so resisted be of a not inhumane temper, and the resisting one perfectly harmless in his passivity; then, in the better moods of the former, he will endeavor charitably to construe to his imagination what proves impossible to be solved by his judgment. Even so, for the most part, I regarded Bartleby and his ways. Poor fellow! thought I, he means no mischief; it is plain he intends no insolence; his aspect sufficiently evinces that his eccentricities are involuntary. He is useful to me. I can get along with him. If I turn him away, the chances are he will fall in with some less indulgent employer, and then he will be rudely treated, and perhaps driven forth miserably to starve. Yes. Here I can cheaply purchase a delicious self-approval. To befriend Bartleby; to humor him in his strange willfulness, will cost me little or nothing, while I lay up in my soul what will eventually prove a sweet morsel for my conscience. But this mood was not invariable with me. The passiveness of Bartleby sometimes irritated me. I felt strangely goaded on to encounter him in new opposition, to elicit some angry spark from him answerable to my own. But indeed I might as well have essayed to strike fire with my knuckles against a bit of Windsor soap. But one afternoon the evil impulse in me mastered me, and the following little scene ensued:

      "Bartleby," said I, "when those papers are all copied, I will compare them with you."

      "I would prefer not to."

      "How? Surely you do not mean to persist in that mulish vagary?"

      No answer.

      I threw open the folding-doors near by, and turning upon Turkey and Nippers, exclaimed in an excited manner--

      "He says, a second time, he won't examine his papers. What do you think of it, Turkey?"

      It was afternoon, be it remembered. Turkey sat glowing like a brass boiler, his bald head steaming, his hands reeling among his blotted papers.

      "Think of it?" roared Turkey; "I think I'll just step behind his screen, and black his eyes for him!"

      So saying, Turkey rose to his feet and threw his arms into a pugilistic position. He was hurrying away to make good his promise, when I detained him, alarmed at the effect of incautiously rousing Turkey's combativeness after dinner.

      "Sit down, Turkey," said I, "and hear what Nippers has to say. What do you think of it, Nippers? Would I not be justified in immediately dismissing Bartleby?"

      "Excuse me, that is for you to decide, sir. I think his conduct quite unusual, and indeed unjust, as regards Turkey and myself. But it may only be a passing whim."

      "Ah," exclaimed I, "you have strangely changed your mind then--you speak very gently of him now."

      "All beer," cried Turkey; "gentleness is effects of beer--Nippers and I dined together to-day. You see how gentle I am, sir. Shall I go and black his eyes?"

      "You refer to Bartleby, I suppose. No, not to-day, Turkey," I replied; "pray, put up your fists."

      I closed the doors, and again advanced towards Bartleby. I felt additional incentives tempting me to my fate. I burned to be rebelled against again. I remembered that Bartleby never left the office.

      "Bartleby," said I, "Ginger Nut is away; just step round to the Post Office, won't you? (it was but a three minutes walk,) and see if there is any thing for me."

      "I would prefer not to."

      "You will not?"

      "I prefer not."

      I staggered to my desk, and sat there in a deep study. My blind inveteracy returned. Was there any other thing in which I could procure myself to be ignominiously repulsed by this lean, penniless with?--my hired clerk? What added thing is there, perfectly reasonable, that he will be sure to refuse to do?

      "Bartleby!"

      No answer.

      "Bartleby," in a louder tone.

      No answer.

      "Bartleby," I roared.

      Like a very ghost, agreeably to the laws of magical invocation, at the third summons, he appeared at the entrance of his hermitage.

      "Go to the next room, and tell Nippers to come to me."

      "I prefer not to," he respectfully and slowly said, and mildly disappeared.

      "Very good, Bartleby," said I, in a quiet sort of serenely severe self-possessed tone, intimating the unalterable purpose of some terrible retribution very close at hand. At the moment I half intended something of the kind. But upon the whole, as it was drawing towards my dinner-hour, I thought it best to put on my hat and walk home for the day, suffering much from perplexity and distress of mind.

      Shall I acknowledge it? The conclusion of this whole business was that it soon became a fixed fact of my chambers, that a pale young scrivener, by the name of Bartleby, had a desk there; that he copied for me at the usual rate of four cents a folio (one hundred words); but he was permanently exempt from examining the work done by him, that duty being transferred to Turkey and Nippers, one of compliment doubtless to their superior acuteness; moreover, said Bartleby was never on any account to be dispatched on the most trivial errand of any sort; and that even if entreated to take upon him such a matter, it was generally understood that he would prefer not to--in other words, that he would refuse point-blank.

      32 As days passed on, I became considerably reconciled to Bartleby. His steadiness, his freedom from all dissipation, his incessant industry (except when he chose to throw himself into a standing revery behind his screen), his great stillness, his unalterableness of demeanor under all circumstances, made him a valuable acquisition. One prime thing was this,--he was always there;--first in the morning, continually through the day, and the last at night. I had a singular confidence in his honesty. I felt my most precious papers perfectly safe in his hands. Sometimes to be sure I could not, for the very soul of me, avoid falling into sudden spasmodic passions with him. For it was exceeding difficult to bear in mind all the time those strange peculiarities, privileges, and unheard of exemptions, forming the tacit stipulations on Bartleby's part under which he remained in my office. Now and then, in the eagerness of dispatching pressing business, I would inadvertently summon Bartleby, in a short, rapid tone, to put his finger, say, on the incipient tie of a bit of red tape with which I was about compressing some papers. Of course, from behind the screen the usual answer, "I prefer not to," was sure to come; and then, how could a human creature with the common infirmities of our nature, refrain from bitterly exclaiming upon such perverseness--such unreasonableness. However, every added repulse of this sort which I received only tended to lessen the probability of my repeating the inadvertence.

      Here is must be said, that according to the custom of most legal gentlemen occupying chambers in densely-populated law buildings, there were several keys to my door. One was kept by a woman residing in the attic, which person weekly scrubbed and daily swept and dusted my apartments. Another was kept by Turkey for convenience sake. The third I sometimes carried in my own pocket. The fourth I knew not who had.

      Now, one Sunday morning I happened to go to Trinity Church, to hear a celebrated preacher, and finding myself rather early on the ground, I thought I would walk round to my chambers for a while. Luckily I had my key with me; but upon applying it to the lock, I found it resisted by something inserted from the inside. Quite surprised, I called out; when to my consternation a key was turned from within; and thrusting his lean visage at me, and holding the door ajar, the apparition of Bartleby appeared, in his shirt sleeves, and otherwise in a strangely tattered dishabille, saying quietly that he was sorry, but he was deeply engaged just then, and--preferred not admitting me at present. In a brief word or two, he moreover added, that perhaps I had better walk round the block two or three times, and by that time he would probably have concluded his affairs. Now, the utterly unsurmised appearance of Bartleby, tenanting my law-chambers of a Sunday morning, with his cadaverously gentlemanly nonchalance, yet withal firm and self-possessed, had such a strange effect upon me, that incontinently I slunk away from my own door, and did as desired. But not without sundry twinges of impotent rebellion against the mild effrontery of this unaccountable scrivener. Indeed, it was his wonderful mildness chiefly, which not only disarmed me, but unmanned me, as it were. For I consider that one, for the time, is a sort of unmanned when he tranquilly permits his hired clerk to dictate to him, and order him away from his own premises. Furthermore, I was full of uneasiness as to what Bartleby could possibly be doing in my office in his shirt sleeves, and in an otherwise dismantled condition of a Sunday morning. Was any thing amiss going on? Nay, that was out of the question. It was not to be thought of for a moment that Bartleby was an immoral person. But what could he be doing there?--copying? Nay again, whatever might be his eccentricities, Bartleby was an eminently decorous person. He would be the last man to sit down to his desk in any state approaching to nudity. Besides, it was Sunday; and there was something about Bartleby that forbade the supposition that we would by any secular occupation violate the proprieties of the day.

      Nevertheless, my mind was not pacified; and full of a restless curiosity, at last I returned to the door. Without hindrance I inserted my key, opened it, and entered. Bartleby was not to be seen. I looked round anxiously, peeped behind his screen; but it was very plain that he was gone. Upon more closely examining the place, I surmised that for an indefinite period Bartleby must have ate, dressed, and slept in my office, and that too without plate, mirror, or bed. The cushioned seat of a rickety old sofa in one corner bore t faint impress of a lean, reclining form. Rolled away under his desk, I found a blanket; under the empty grate, a blacking box and brush; on a chair, a tin basin, with soap and a ragged towel; in a newspaper a few crumbs of ginger-nuts and a morsel of cheese. Yet, thought I, it is evident enough that Bartleby has been making his home here, keeping bachelor's hallall by himself. Immediately then the thought came sweeping across me, What miserable friendlessness and loneliness are here revealed! His poverty is great; but his solitude, how horrible! Think of it. Of a Sunday, Wall-street is deserted as Petra; and every night of every day it is an emptiness. This building too, which of week-days hums with industry and life, at nightfall echoes with sheer vacancy, and all through Sunday is forlorn. And here Bartleby makes his home; sole spectator of a solitude which he has seen all populous--a sort of innocent and transformed Marius brooding among the ruins of Carthage!

      For the first time in my life a feeling of overpowering stinging melancholy seized me. Before, I had never experienced aught but a not-unpleasing sadness. The bond of a common humanity now drew me irresistibly to gloom. A fraternal melancholy! For both I and Bartleby were sons of Adam. I remembered the bright silks and sparkling faces I had seen that day in gala trim, swan-like sailing down the Mississippi of Broadway; and I contrasted them with the pallid copyist, and thought to myself, Ah, happiness courts the light, so we deem the world is gay; but misery hides aloof, so we deem that misery there is none. These sad fancyings-- chimeras, doubtless, of a sick and silly brain--led on to other and more special thoughts, concerning the eccentricities of Bartleby. Presentiments of strange discoveries hovered round me. The scrivener's pale form appeared to me laid out, among uncaring strangers, in its shivering winding sheet.

      Suddenly I was attracted by Bartleby's closed desk, the key in open sight left in the lock.

      I mean no mischief, seek the gratification of no heartless curiosity, thought I; besides, the desk is mine, and its contents too, so I will make bold to look within. Every thing was methodically arranged, the papers smoothly placed. The pigeon holes were deep, and removing the files of documents, I groped into their recesses. Presently I felt something there, and dragged it out. It was an old bandanna handkerchief, heavy and knotted. I opened it, and saw it was a savings' bank.

      I now recalled all the quiet mysteries which I had noted in the man. I remembered that he never spoke but to answer; that though at intervals he had considerable time to himself, yet I had never seen him reading--no, not even a newspaper; that for long periods he would stand looking out, at his pale window behind the screen, upon the dead brick wall; I was quite sure he never visited any refectory or eating house; while his pale face clearly indicated that he never drank beer like Turkey, or tea and coffee even, like other men; that he never went any where in particular that I could learn; never went out for a walk, unless indeed that was the case at present; that he had declined telling who he was, or whence he came, or whether he had any relatives in the world; that though so thin and pale, he never complained of ill health. And more than all, I remembered a certain unconscious air of pallid--how shall I call it?--of pallid haughtiness, say, or rather an austere reserve about him, which had positively awed me into my tame compliance with his eccentricities, when I had feared to ask him to do the slightest incidental thing for me, even though I might know, from his long-continued motionlessness, that behind his screen he must be standing in one of those dead-wall reveries of his.

      Revolving all these things, and coupling them with the recently discovered fact that he made my office his constant abiding place and home, and not forgetful of his morbid moodiness; revolving all these things, a prudential feeling began to steal over me. My first emotions had been those of pure melancholy and sincerest pity; but just in proportion as the forlornness of Bartleby grew and grew to my imagination, did that same melancholy merge into fear, that pity into repulsion. So true it is, and so terrible too, that up to a certain point the thought or sight of misery enlists our best affections; but, in certain special cases, beyond that point it does not. They err who would assert that invariably this is owing to the inherent selfishness of the human heart. It rather proceeds from a certain hopelessness of remedying excessive and organic ill. To a sensitive being, pity is not seldom pain. And when at last it is perceived that such pity cannot lead to effectual succor, common sense bids the soul be rid of it. What I saw that morning persuaded me that the scrivener was the victim of innate and incurable disorder. I might give alms to his body; but his body did not pain him; it was his soul that suffered, and his soul I could not reach.

      I did not accomplish the purpose of going to Trinity Church that morning. Somehow, the things I had seen disqualified me for the time from church-going. I walked homeward, thinking what I would do with Bartleby. Finally, I resolvedupon this;--I would put certain calm questions to him the next morning, touching his history, &c., and if he declined to answer then openly and reservedly (and I supposed he would prefer not), then to give him a twenty dollar bill over and above whatever I might owe him, and tell him his services were no longer required; but that if in any other way I could assist him, I would be happy to do so, especially if he desired to return to his native place, wherever that might be, I would willingly help to defray the expenses. Moreover, if after reaching home, he found himself at any time in want of aid, a letter from him would be sure of a reply.

      The next morning came.

      "Bartleby," said I, gently calling to him behind the screen.

      No reply.

      "Bartleby," said I, in a still gentler tone, "come here; I am not going to ask you to do any thing you would prefer not to do--I simply wish to speak to you."

      Upon this he noiselessly slid into view.

      "Will you tell me, Bartleby, where you were born?"

      "I would prefer not to."

      "Will you tell me anything about yourself?"

      "I would prefer not to."

      "But what reasonable objection can you have to speak to me? I feel friendly towards you."

      He did not look at me while I spoke, but kept his glance fixed upon my bust of Cicero, which as I then sat, was directly behind me, some six inches above my head. "What is your answer, Bartleby?" said I, after waiting a considerable time for a reply, during which his countenance remained immovable, only there was the faintest conceivable tremor of the white attenuated mouth.

      "At present I prefer to give no answer," he said, and retired into his hermitage.

      It was rather weak in me I confess, but his manner on this occasion nettled me. Not only did there seem to lurk in it a certain disdain, but his perverseness seemed ungrateful, considering the undeniable good usage and indulgence he had received from me.

      Again I sat ruminating what I should do.Mortified as I was at his behavior, and resolved as I had been to dismiss him when I entered my office, nevertheless I strangely felt something superstitious knocking at my heart, and forbidding me to carry out my purpose, and denouncing me for a villain if I dared to breathe one bitter word against this forlornest of mankind. At last, familiarly drawing my chair behind his screen, I sat down and said: "Bartleby, never mind then about revealing your history; but let me entreat you, as a friend, to comply as far as may be with the usages of this office. Say now you will help to examine papers tomorrow or next day: in short, say now that in a day or two you will begin to be a little reasonable:--say so, Bartleby."

      "At present I would prefer not to be a little reasonable was his idly cadaverous reply.,"

      Just then the folding-doors opened, and Nippers approached. He seemed suffering from an unusually bad night's rest, induced by severer indigestion than common. He overheard those final words of Bartleby.

      "Prefer not, eh?" gritted Nippers--"I'd prefer him, if I were you, sir," addressing me--"I'd prefer him; I'd give him preferences, the stubborn mule! What is it, sir, pray, that he prefers not to do now?"

      Bartleby moved not a limb.

      "Mr. Nippers," said I, "I'd prefer that you would withdraw for the present."

      Somehow, of late I had got into the way of involuntary using this word "prefer" upon all sorts of not exactly suitable occasions. And I trembled to think that my contact with the scrivener had already and seriously affected me in a mental way. And what further and deeper aberration might it not yet produce? This apprehension had not been without efficacy in determining me to summary means.

      As Nippers, looking very sour and sulky, was departing, Turkey blandly and deferentially approached.

      "With submission, sir," said he, "yesterday I was thinking about Bartleby here, and I think that if he would but prefer to take a quart of good ale every day, it would do much towards mending him, and enabling him to assist in examining his papers."

      "So you have got the word too," said I, slightly excited.

      "With submission, what word, sir," asked Turkey, respectfully crowding himself into the contracted space behind the screen, and by so doing, making me jostle the scrivener. "What word, sir?"

      "I would prefer to be left alone here," said Bartleby, as if offended at being mobbed in his privacy.

      "That's the word, Turkey," said I--"that's it."

      "Oh, prefer oh yes--queer word. I never use it myself. But, sir as I was saying, if he would but prefer--"

      "Turkey," interrupted I, "you will please withdraw."

      "Oh, certainly, sir, if you prefer that I should."

      As he opened the folding-door to retire, Nippers at his desk caught a glimpse of me, and asked whether I would prefer to have a certain paper copied on blue paper or white. He did not in the least roguishly accent the word prefer. It was plain that it involuntarily rolled from his tongue. I thought to myself, surely I must get rid of a demented man, who already has in some degree turned the tongues, if not the heads of myself and clerks. But I thought it prudent not to break the dismission at once.

      The next day I noticed that Bartleby did nothing but stand at his window in his dead-wall revery. Upon asking him why he did not write, he said that he had decided upon doing no more writing.

      "Why, how now? what next?" exclaimed I, "do no more writing?"

      "No more."

      "And what is the reason?"

      "Do you not see the reason for yourself," he indifferently replied.

      I looked steadfastly at him, and perceived that his eyes looked dull and glazed. Instantly it occurred to me, that his unexampled diligence in copying by his dim window for the first few weeks of his stay with me might have temporarily impaired his vision.

      I was touched. I said something in condolence with him. I hinted that of course he did wisely in abstaining from writing for a while; and urged him to embrace that opportunity of taking wholesome exercise in the open air. This, however, he did not do. A few days after this, my other clerks being absent, and being in a great hurry to dispatch certain letters by the mail, I thought that, having nothing else earthly to do, Bartleby would surely be less inflexible than usual, and carry these letters to the post-office. But he blankly declined. So, much to my inconvenience, I went myself.

      Still added days went by. Whether Bartleby's eyes improved or not, I could not say. To all appearance, I thought they did. But when I asked him if they did, he vouchsafed no answer. At all events, he would do no copying. At last, in reply to my urgings, he informed me that he had permanently given up copying.

      "What!" exclaimed I; "suppose your eyes should get entirely well- better than ever before--would you not copy then?"

      "I have given up copying," he answered, and slid aside.

      He remained as ever, a fixture in my chamber. Nay--if that were possible--he became still more of a fixture than before. What was to be done? He would do nothing in the office: why should he stay there? In plain fact, he had now become a millstone to me, not only useless as a necklace, but afflictive to bear. Yet I was sorry for him. I speak less than truth when I say that, on his own account, he occasioned me uneasiness. If he would but have named a single relative or friend, I would instantly have written, and urged their taking the poor fellow away to some convenient retreat. But he seemed alone, absolutely alone in the universe. A bit of wreck</font> in the mid Atlantic. At length, necessities connected with my business tyrannized over all other considerations. Decently as I could, I told Bartleby that in six days' time he must unconditionally leave the office. I warned him to take measures, in the interval, for procuring some other abode. I offered to assist him in this endeavor, if he himself would but take the first step towards a removal. "And when you finally quit me, Bartleby," added I, "I shall see that you go not away entirely unprovided. Six days from this hour, remember."

      At the expiration of that period, I peeped behind the screen, and lo! Bartleby was there.

      I buttoned up my coat, balanced myself; advanced slowly towards him, touched his shoulder, and said, "The time has come; you must quit this place; I am sorry for you; here is money; but you must go."

      "I would prefer not," he replied, with his back still towards me.

      "You must."

      He remained silent.

      Now I had an unbounded confidence in this man's common honesty. He had frequently restored to me six pences and shillings carelessly dropped upon the floor, for I am apt to be very reckless in such shirt-button affairs. The proceeding then which followed will not be deemed extraordinary. "Bartleby," said I, "I owe you twelve dollars on account; here are thirty-two; the odd twenty are yours.--Will you take it? and I handed the bills towards him.

      But he made no motion.

      "I will leave them here then," putting them under a weight on the table. Then taking my hat and cane and going to the door I tranquilly turned and added--"After you have removed your things from these offices, Bartleby, you will of course lock the door--since every one is now gone for the day but you--and if you please, slip your key underneath the mat, so that I may have it in the morning. I shall not see you again; so good-bye to you. If hereafter in your new place of abode I can be of any service to you, do not fail to advise me by letter. Good-bye, Bartleby, and fare you well."

      But he answered not a word; like the last column of some ruined temple, he remained standing mute and solitary in the middle of the otherwise deserted room.

      As I walked home in a pensive mood, my vanity got the better of my pity. I could not but highly plume myself on my masterly management in getting rid of Bartleby. Masterly I call it, and such it must appear to any dispassionate thinker. The beauty of my procedure seemed to consist in its perfect quietness. There was no vulgar bullying, no bravado of any sort, no choleric hectoring and striding to and fro across the apartment, jerking out vehement commands for Bartleby to bundle himself off with his beggarly traps. Nothing of the kind. Without loudly bidding Bartleby depart--as an inferior genius might have done--I assumed the ground that depart he must; and upon the assumption built all I had to say. The more I thought over my procedure, the more I was charmed with it. Nevertheless, next morning, upon awakening, I had my doubts,--I had somehow slept off the fumes of vanity. One of the coolest and wisest hours a man has, is just after he awakes in the morning. My procedure seemed as sagacious as ever,--but only in theory. How it would prove in practice--there was the rub. It was truly a beautiful thought to have assumed Bartleby's departure; but, after all, that assumption was simply my own, and none of Bartleby's. The great point was, not whether I had assumed that he would quit me, but whether he would prefer so to do. He was more a man of preferences than assumptions.

      After breakfast, I walked down town, arguing the probabilities pro and con. One moment I thought it would prove a miserable failure, and Bartleby would be found all alive at my office as usual; the next moment it seemed certain that I should see his chair empty. And so I kept veering about. At the corner of Broadway and Canal- street, I saw quite an excited group of people standing in earnest conversation.

      "I'll take odds he doesn't," said a voice as I passed.

      "Doesn't go?--done!" said I, "put up your money."

      I was instinctively putting my hand in my pocket to produce my own, when I remembered that this was an election day. The words I had overheard bore no reference to Bartleby, but to the success or non-success of some candidate for the mayoralty. In my intent frame of mind, I had, as it were, imagined that all Broadway shared in my excitement, and were debating the same question with me. I passed on, very thankful that the uproar of the street screened my momentary absent-mindedness.

      As I had intended, I was earlier than usual at my office door. I stood listening for a moment. All was still. He must be gone. I tried the knob. The door was locked. Yes, my procedure had worked to a charm; he indeed must be vanished. Yet a certain melancholy mixed with this: I was almost sorry for my brilliant success. I was fumbling under the door mat for the key, which Bartleby was to have left there for me, when accidentally my knee knocked against a panel, producing a summoning sound, and in response a voice came to me from within--"Not yet; I am occupied."

      It was Bartleby.

      I was thunderstruck. For an instant I stood like the man who, pipe in mouth, was killed one cloudless afternoon long ago in Virginia, by summer lightning; at his own warm open window he was killed, and remained leaning out there upon the dreamy afternoon, till some one touched him, when he fell. "Not gone!" I murmured at last. But again obeying that wondrous ascendancy which the inscrutable scrivener had over me, and from which ascendancy, for all my chafing, I could not completely escape, I slowly went down stairs and out into the street, and while walking round the block, considered what I should next do in this unheard-of-perplexity. Turn the man out by an actual thrusting I could not; to drive him away by calling him hard names would not do; calling in the police was an unpleasant idea; and yet, permit him to enjoy his cadaverous triumph over me,--this too I could not think of. What was to be done? or, if nothing could be done, was there any thing further that I could assume in the matter? Yes, as before I had prospectively assumed that Bartleby would depart, so now I might retrospectively assume that departed he was. In the legitimate carrying out of this assumption, I might enter my office in a great hurry, and pretending not to see Bartleby at all, walk straight against him as if he were air. Such a proceeding would in a singular degree have the appearance of a home-thrust. It was hardly possible that Bartleby could withstand such an application of the doctrine of assumptions. But upon second thoughts the success of the plan seemed rather dubious. I resolved to argue the matter over with him again.

      Bartleby," said I, entering the office, with a quietly severe expression. "I am seriously displeased. I am pained, Bartleby. I had thought better of you. I had imagined you of such a gentlemanly organization, that in any delicate dilemma a slight hint would suffice--in short, an assumption. But it appears I am deceived. Why," I added, unaffectedly starting, "you have not even touched the money yet," pointing to it, just where I had left it the evening previous.

      He answered nothing.

      "Will you, or will you not, quit me?" I now demanded in a sudden passion, advancing close to him.

      "I would prefer not to quit you," he replied, gently emphasizing the not.

      "What earthly right have you to stay here? do you pay any rent? Do you pay my taxes? Or is this property yours?"

      He answered nothing.

      "Are you ready to go on and write now? Are your eyes recovered? Could you copy a small paper for me this morning? or help examine a few lines? or step round to the post-office? In a word, will you do any thing at all, to give a coloring to your refusal to depart the premises?"

      He silently retired into his hermitage.

      I was now in such a state of nervous resentment that I thought it but prudentto check myself at present from further demonstrations. Bartleby and I were alone. I remembered the tragedy of the unfortunate Adams and the still more unfortunate Colt in the solitary office of the latter; and how poor Colt, being dreadfully incensed by Adams, and imprudently permitting himself to get wildly excited, was at unawares hurried into his fatal act--an act which certainly no man could possibly deplore more than the actor himself. Often it had occurred to me in my ponderings upon the subject, that had that altercation taken place in the public street, or at a private residence, it would not have terminated as it did. It was the circumstance of being alone in a solitary office, up stairs, of a building entirely unhallowed by humanizing domestic associations--an uncarpeted office, doubtless of a dusty, haggard sort of appearance;--this it must have been, which greatly helped to enhance the irritable desperation of the hapless Colt.

      But when this old Adam of resentment rose in me and tempted me concerning Bartleby, I grappled him and threw him. How? Why, simply by recalling the divine injunction: "A new commandment give I unto you, that ye love one another." Yes, this it was that saved me. Aside from higher considerations, charity often operates as a vastly wise and prudent principle--a great safeguard to its possessor. Men have committed murder for jealousy's sake, and anger's sake, and hatred's sake, and selfishness' sake, and spiritual pride's sake; but no man that ever I heard of, ever committed a diabolical murder for sweet charity's sake. Mere self-interest, then, if no better motive can be enlisted, should, especially with high-tempered men, prompt all beings to charity and philanthropy. At any rate, upon the occasion in question, I strove to drown my exasperated feelings towards the scrivener by benevolently construing his conduct. Poor fellow, poor fellow! thought I, he don't mean any thing; and besides, he has seen hard times, and ought to be indulged.

      I endeavored also immediately to occupy myself, and at the same time to comfort my despondency.I tried to fancy that in the course of the morning, at such time as might prove agreeable to him, Bartleby, of his own free accord, would emerge from his hermitage, and take up some decided line of march in the direction of the door. But no. Half-past twelve o'clock came; Turkey began to glow in the face, overturn his inkstand, and become generally obstreperous; Nippers abated down into quietude and courtesy; Ginger Nut munched his noon apple; and Bartleby remained standing at his window in one of his profoundest deadwall reveries. Will it be credited? Ought I to acknowledge it? That afternoon I left the office without saying one further word to him.

      Some days now passed, during which, at leisure intervals I looked a little into Edwards on the Will," and "Priestly on Necessity." Under the circumstances, those books induced a salutary feeling. Gradually I slid into the persuasion that these troubles of mine touching the scrivener, had been all predestinated from eternity, and Bartleby was billeted upon me for some mysterious purpose of an all-wise Providence, which it was not for a mere mortal like me to fathom. Yes, Bartleby, stay there behind your screen, thought I; I shall persecute you no more; you are harmless and noiseless as any of these old chairs; in short, I never feel so private as when I know you are here. At least I see it, I feel it; I penetrate to the predestinated purpose of my life. I am content. Others may have loftier parts to enact; but my mission in this world, Bartleby, is to furnish you with office-room for such period as you may see fit to remain.

      I believe that this wise and blessed frame of mind would have continued with me, had it not been for the unsolicited and uncharitable remarks obtruded upon me by my professional friends who visited the rooms. But thus it often is, that the constant friction of illiberal minds wears out at last the best resolves of the more generous. Though to be sure, when I reflected upon it, it was not strange that people entering my office should be struck by the peculiar aspect of the unaccountable Bartleby, and so be tempted to throw out some sinister observations concerning him. Sometimes an attorney having business with me, and calling at my office, and finding no one but the scrivener there, would undertake to obtain some sort of precise information from him touching my whereabouts; but without heeding his idle talk, Bartleby would remain standing immovable in the middle of the room. So after contemplating him in that position for a time, the attorney would depart, no wiser than he came.

      Also, when a Reference was going on, and the room full of lawyers and witnesses and business was driving fast; some deeply occupied legal gentleman present, seeing Bartleby wholly unemployed, would request him to run round to his (the legal gentleman's) office and fetch some papers for him. Thereupon, Bartleby would tranquilly decline, and remain idle as before. Then the lawyer would give a great stare, and turn to me. And what could I say? At last I was made aware that all through the circle of my professional acquaintance, a whisper of wonder was running round, having reference to the strange creature I kept at my office. This worried me very much. And as the idea came upon me of his possibly turning out a long-lived man, and keep occupying my chambers, and denying my authority; and perplexing my visitors; and scandalizing my professional reputation; and casting a general gloom over the premises; keeping soul and body together to the last upon his savings (for doubtless he spent but half a dime a day), and in the end perhaps outlive me, and claim possession of my office by right of his perpetual occupancy: as all these dark anticipations crowded upon me more and more, and my friends continually intruded their relentless remarks upon the apparition in my room; a great change was wrought in me. I resolved to gather all my faculties together, and for ever rid me of this intolerable incubus.

      Ere revolving any complicated project, however, adapted to this end, I first simply suggested to Bartleby the propriety of his permanent departure. In a calm and serious tone, I commended the idea to his careful and mature consideration. But having taken three days to meditate upon it, he apprised me that his original determination remained the same; in short, that he still preferred to abide with me.

      What shall I do? I now said to myself, buttoning up my coat to the last button. What shall I do? what ought I to do? what does conscience say I should do with this man, or rather ghost. Rid myself of him, I must; go, he shall. But how? You will not thrust him, the poor, pale, passive mortal,--you will not thrust such a helpless creature out of your door? you will not dishonor yourself by such cruelty? No, I will not, I cannot do that. Rather would I let him live and die here, and then mason up his remains in the wall. What then will you do? For all your coaxing, he will not budge. Bribes he leaves under your own paperweight on your table; in short, it is quite plain that he prefers to cling to you.

      Then something severe, something unusual must be done. What! surely you will not have him collared by a constable, and commit his innocent pallor to the common jail? And upon what ground could you procure such a thing to be done?--a vagrant, is he? What! he a vagrant, a wanderer, who refuses to budge? It is because he will not be a vagrant, then, that you seek to count him as a vagrant. That is too absurd. No visible means of support: there I have him. Wrong again: for indubitably he does support himself, and that is the only unanswerable proof that any man can show of his possessing the means so to do. No more then. Since he will not quit me, I must quit him. I will change my offices; I will move elsewhere; and give him fair notice, that if I find him on my new premises I will then proceed against him as a common trespasser.

      Acting accordingly, next day I thus addressed him: "I find these chambers too far from the City Hall; the air is unwholesome. In a word, I propose to remove my offices next week, and shall no longer require your services. I tell you this now, in order that you may seek another place."

      He made no reply, and nothing more was said.

      On the appointed day I engaged carts and men, proceeded to my chambers, and having but little furniture, every thing was removed in a few hours. Throughout, the scrivener remained standing behind the screen, which I directed to be removed the last thing. It was withdrawn; and being folded up like a huge folio, left him the motionless occupant of a naked room. I stood in the entry watching him a moment, while something from within me upbraided me.

      I re-entered, with my hand in my pocket--and--and my heart in my mouth.

      "Good-bye, Bartleby; I am going--good-bye, and God some way bless you; and take that," slipping something in his hand. But it dropped to the floor, and then,--strange to say--I tore myself from him whom I had so longed to be rid of.

      Established in my new quarters, for a day or two I kept the door locked, and started at every footfall in the passages. When I returned to my rooms after any little absence, I would pause at the threshold for an instant, and attentively listen, ere applying my key. But these fears were needless. Bartleby never came nigh me.

      I thought all was going well, when a perturbed looking stranger visited me, inquiring whether I was the person who had recently occupied rooms at No.--Wall-street.

      Full of forebodings, I replied that I was.

      "Then, sir," said the stranger, who proved a lawyer, "you are responsible for the man you left there. He refuses to do any copying; he refuses to do any thing; he says he prefers not to; and he refuses to quit the premises."

      "I am very sorry, sir," said I, with assumed tranquillity, but an inward tremor, "but, really, the man you allude to is nothing to me --he is no relation or apprentice of mine, that you should hold me responsible for him."

      "In mercy's name, who is he?"

      "I certainly cannot inform you. I know nothing about him. Formerly I employed him as a copyist; but he has done nothing for me now for some time past."

      "I shall settle him then,--good morning, sir."

      Several days passed, and I heard nothing more; and though I often felt a charitable prompting to call at the place and see poor Bartleby, yet a certain squeamishness of I know not what withheld me.

      All is over with him, by this time, thought I at last, when through another week no further intelligence reached me. But coming to my room the day after, I found several persons waiting at my door in a high state of nervous excitement.

      "That's the man--here he comes," cried the foremost one, whom recognized as the lawyer who had previously called upon me alone.

      "You must take him away, sir, at once," cried a portly person among them, advancing upon me, and whom I knew to be the landlord of No.--Wall-street. "These gentlemen, my tenants, cannot stand it any longer; Mr. B--" pointing to the lawyer, "has turned him out of his room, and he now persists in haunting the buildinggenerally, sitting upon the banisters of the stairs by day, and sleeping in the entry by night. Every body is concerned; clients are leaving the offices; some fears are entertained of a mob; something you must do, and that without delay."

      Aghast at this torment, I fell back before it, and would fain have locked myselfin my new quarters. In vain I persisted that Bartleby was nothing to me--no more than to any one else. In vain:--I was the last person known to have any thing to do with him, and they held me to the terrible account. Fearful then of being exposed in the papers (as one person present obscurely threatened) I considered the matter, and at length said, that if the lawyer would give me a confidential interview with the scrivener, in his (the lawyer's) own room, I would that afternoon strive my best to rid them of the nuisance they complained of.

      Going up stairs to my old haunt, there was Bartleby silently sitting upon the banister at the landing.

      "What are you doing here, Bartleby?" said I.

      "Sitting upon the banister," he mildly replied.

      I motioned him into the lawyer's room, who then left us.

      "Bartleby," said I, "are you aware that you are the cause of great tribulation to me, by persisting in occupying the entry after being dismissed from the office?"

      No answer.

      "Now one of two things must take place. Either you must do something or something must be done to you. Now what sort of business would you like to engage in? Would you like to re-engage in copying for some one?"

      "No; I would prefer not to make any change."

      "Would you like a clerkship in a dry-goods store?"

      "There is too much confinement about that. No, I would not like a clerkship; but I am not particular."

      "Too much confinement," I cried, "why you keep yourself confined all the time!"

      "I would prefer not to take a clerkship," he rejoined, as if to settle that little item at once.

      "How would a bar-tender's business suit you? There is no trying of the eyesight in that."

      "I would not like it at all; though, as I said before, I am not particular."

      His unwonted wordiness inspirited me. I returned to the charge.

      "Well then, would you like to travel through the country collecting bills for the merchants? That would improve your health."

      "No, I would prefer to be doing something else."

      "How then would going as a companion to Europe, to entertain some young gentleman with your conversation,--how would that suit you?"

      "Not at all. It does not strike me that there is any thing definite about that. I like to be stationary. But I am not particular.

      "Stationary you shall be then," I cried, now losing all patience, and for the first time in all my exasperating connection with him fairly flying into a passion. "If you do not go away from these premises before night, I shall feel bound--indeed I am bound--to-- to--to quit the premises myself!" I rather absurdly concluded, knowing not with what possible threat to try to frighten his immobility into compliance. Despairing of all further efforts, I was precipitately leaving him, when a final thought occurred to me--one which had not been wholly unindulged before.

      "Bartleby," said I, in the kindest tone I could assume under such exciting circumstances, "will you go home with me now--not to my office, but my dwelling--and remain there till we can conclude upon some convenient arrangement for you at our leisure? Come, let us start now, right away."

      "No: at present I would prefer not to make any change at all."

      I answered nothing; but effectualy dodging every one by the suddenness and rapidity of my flight, rushed from the building, ran up Wall-street towards Broadway, and jumping into the first omnibus was soon removed from pursuit. As soon as tranquility returned I distinctly perceived that I had now done all that I possibly could, both in respect to the demands of the landlord and his tenants, and with regard to my own desire and sense of duty, to benefit Bartleby, and shield him from rude persecution. I now strove to be entirely care-free and quiescent; and my conscience justified me in the attempt; though indeed it was not so successful as I could have wished. So fearful was I of being again hunted out by the incensed landlord and his exasperated tenants, that, surrendering my business to Nippers, for a few days I drove about the upper part of the town and through the suburbs, in my rockaway; crossed over to Jersey City and Hoboken, and paid fugitive visits to Manhattanville and Astoria. In fact I almost lived in my rockaway for the time.

      When again I entered my office, lo, a note from the landlord lay upon desk. opened it with trembling hands. informed me that writer had sent to police, and Bartleby removed the Tombs as a vagrant. Moreover, since I knew more about him than any one else, he wished me to appear at that place, and make a suitable statement of the facts. These tidings had a conflicting effect upon me. At first I was indignant; but at last almost approved. The landlord's energetic, summary disposition, had led him to adopt a procedure which I do not think I would have decided upon myself; and yet as a last resort, under such peculiar circumstances, it seemed the only plan.

      As I afterwards learned, the poor scrivener, when told that he must be conducted to the Tombs, offered not the slightest obstacle, but in his pale unmoving way, silently acquiesced.

      Some of the compassionate and curious bystanders joined the party; and headed by one of the constables arm in arm with Bartleby, the silent procession filed its way through all the noise, and heat, and joy of the roaring thoroughfares at noon.

      The same day I received the note I went to the Tombs, or to speak more properly, the Halls of Justice. Seeking the right officer, I stated the purpose of my call, and was informed that the individual I described was indeed within. I then assured the functionary that Bartleby was a perfectly honest man, and greatly to be compassionated, however unaccountably eccentric. I narrated all I knew,and closed by suggesting the idea of letting him remain in as indulgent confinement as possible till something less harsh might be done--though indeed I hardly knew what. At all events, if nothing else could be decided upon, the alms-house must receive him. I then begged to have an interview.

      Being under no disgraceful charge, and quite serene and harmless in all his ways, they had permitted him freely to wander about the prison, and especially in the inclosed grass-platted yards thereof. And so I found him there, standing all alone in the quietest of the yards, his face towards a high wall, while all around, from the narrow slits of the jail windows, I thought I saw peering out upon him the eyes of murderers and thieves.

      "Bartleby!"

      "I know you," he said, without looking round,--"and I want nothing to say to you."

      "It was not I that brought you here, Bartleby," said I, keenly pained at his implied suspicion. "And to you, this should not be so vile a place. Nothing reproachful attaches to you by being here. And see, it is not so sad a place as one might think. Look, there is the sky, and here is the grass."

      "I know where I am," he replied, but would say nothing more, and so I left him.

      As I entered the corridor again, a broad meat-like man in an apron, accosted me, and jerking his thumb over his shoulder said--"Is that your friend?"

      "Yes."

      "Does he want to starve? If he does, let him live on the prison fare, that's all.

      "Who are you?" asked I, not knowing what to make of such an unofficially speaking person in such a place.

      "I am the grub-man. Such gentlemen as have friends here, hire me to provide them with something good to eat."

      "Is this so?" said I, turning to the turnkey.

      He said it was.

      "Well then," said I, slipping some silver into the grub-man's hands (for so they called him). "I want you to give particular attention to my friend there; let him have the best dinner you can get. And you must be as polite to him as possible."

      "Introduce me, will you?" said the grub-man, looking at me with an expression which seemed to say he was all impatience for an opportunity to give a specimen of his breeding.

      Thinking it would prove of benefit to the scrivener, I acquiesced; and asking the grub-man his name, went up with him to Bartleby.

      "Bartleby, this is a friend; you will find him very useful to you."

      "Your sarvant, sir, your sarvant," said the grub-man, making a low salutation behind his apron. "Hope you find it pleasant here, sir;--spacious grounds--cool apartments, sir--hope you'll stay with us some time--try to make it agreeable. What will you have for dinner today?"

      "I prefer not to dine to-day," said Bartleby, turning away. "It would disagree with me; I am unused to dinners." So saying he slowly moved to the other side of the inclosure, and took up a position fronting the dead-wall.

      "How's this?" said the grub-man, addressing me with a stare of astonishment. "He's odd, aint he?"

      "I think he is a little deranged," said I, sadly.

      "Deranged? deranged is it? Well now, upon my word, I thought that friend of yourn was a gentleman forger; they are always pale and genteel-like, them forgers. I can't help pity 'em--can't help it, sir. Did you know Monroe Edwards?" he added touchingly, and paused. Then, laying his hand pityingly on my shoulder, sighed, "he died of consumption at Sing-Sing. so you weren't acquainted with Monroe?"

      "No, I was never socially acquainted with any forgers. But I cannot stop longer. Look to my friend yonder. You will not lose by it. I will see you again."

      Some few days after this, I again obtained admission to the Tombs, and went through the corridors in quest of Bartleby; but without finding him.

      "I saw him coming from his cell not long ago," said a turnkey, "may be he's gone to loiter in the yards."

      So I went in that direction.

      "Are you looking for the silent man?" said another turnkey passing me. "Yonder he lies--sleeping in the yard there. 'Tis not twenty minutes since I saw him lie down."

      The yard was entirely quiet. It was not accessible to the common prisoners. The surrounding walls, of amazing thickness, kept off all sound behind them. The Egyptian character of the masonry weighed upon me with its gloom. But a soft imprisoned turf grew under foot. The heart of the eternal pyramids, it seemed, wherein, by some strange magic, through the clefts, grass-seed, dropped by birds, had sprung.

      Strangely huddled at the base of the wall, his knees drawn up, and lying on his side, his head touching the cold stones, I saw the wasted Bartleby. But nothing stirred. I paused; then went close up to him; stooped over, and saw that his dim eyes were open; otherwise he seemed profoundly sleeping. Something prompted me to touch him. I felt his hand, when a tingling shiver ran up my arm and down my spine to my feet.

      The round face of the grub-man peered upon me now. "His dinner is ready. Won't he dine to-day, either? Or does he live without dining?"

      "Lives without dining," said I, and closed the eyes.

      "Eh!--He's asleep, aint he?"

      "With kings and counsellors," murmured I.

      * * * * * * * *

      There would seem little need for proceeding further in this history. Imagination will readily supply the meagre recital of poor Bartleby's interment. But ere parting with the reader, let me say, that if this little narrative has sufficiently interested him, to awaken curiosity as to who Bartleby was, and what manner of life he led prior to the present narrator's making his acquaintance, I can only reply, that in such curiosity I fully share, but am wholly unable to gratify it. Yet here I hardly know whether I should divulge one little item of rumor, which came to my ear a few months after the scrivener's decease. Upon what basis it rested, I could never ascertain; and hence how true it is I cannot now tell. But inasmuch as this vague report has not been without a certain strange suggestive interest to me, however said, it may prove the same with some others; and so I will briefly mention it. The report was this: that Bartleby had been a subordinate clerk in the Dead Letter Office at Washington, from which he had been suddenly removed by a change in the administration. When I think over this rumor, I cannot adequately express the emotions which seize me. Dead letters! does it not sound like dead men? Conceive a man by nature and misfortune prone to a pallid hopelessness, can any business seem more fitted to heighten it than that of continually handling these dead letters and assorting them for the flames? For by the cart-load they are annually burned. Sometimes from out the folded paper the pale clerk takes a ring:--the bank-note sent in swiftest charity:--he whom it would relieve, nor eats nor hungers any more; pardon for those who died despairing; hope for those who died unhoping; good tidings for those who died stifled by unrelieved calamities. On errands of life, these letters speed to death.

      Ah Bartleby! Ah humanity!

      ================================================ FILE: test/test-pages/clean-links/source.html ================================================ Bartleby the Scrivener Web Study Text   

      Study Webtext

      "Bartleby the Scrivener: A Story of Wall-Street " (1853) 
      Herman Melville

      To the story text without notes

      Prepared by Ann Woodlief, Virginia Commonwealth University

      Click on text in red for hypertext notes and questions
      I am a rather elderly man. The nature of my avocations for the last thirty years has brought me into more than ordinary contact with what would seem an interesting and somewhat singular set of men of whom as yet nothing that I know of has ever been written:-- I mean the law-copyists or scriveners. I have known very many of them, professionally and privately, and if I pleased, could relate divers histories, at which good-natured gentlemen might smile, and sentimental souls might weep. But I waive the biographies of all other scriveners for a few passages in the life of Bartleby, who was a scrivener the strangest I ever saw or heard of. While of other law-copyists I might write the complete life, of Bartleby nothing of that sort can be done. I believe that no materials exist for a full and satisfactory biography of this man. It is an irreparable loss to literature. Bartleby was one of those beings of whom nothing is ascertainable, except from the original sources, and in his case those are very small. What my own astonished eyes saw of Bartleby, that is all I know of him, except, indeed, one vague report which will appear in the sequel.

      Ere introducing the scrivener, as he first appeared to me, it is fit I make some mention of myself, my employees, my business, my chambers, and general surroundings; because some such description is indispensable to an adequate understanding of the chief character about to be presented.

      Imprimis: I am a man who, from his youth upwards, has been filled with a profound conviction that the easiest way of life is the best.. Hence, though I belong to a profession proverbially energetic and nervous, even to turbulence, at times, yet nothing of that sort have I ever suffered to invade my peace. I am one of those unambitious lawyers who never addresses a jury, or in any way draws down public applause; but in the cool tranquillity of a snug retreat, do a snug business among rich men's bonds and mortgages and title-deeds. The late John Jacob Astor, a personage little given to poetic enthusiasm, had no hesitation in pronouncing my first grand point to be prudence; my next, method. I do not speak it in vanity, but simply record the fact, that I was not unemployed in my profession by the last John Jacob Astor; a name which, I admit, I love to repeat, for it hath a rounded and orbicular sound to it, and rings like unto bullion. I will freely add, that I was not insensible to the late John Jacob Astor's good opinion.

      Some time prior to the period at which this little history begins, my avocations had been largely increased. The good old office, now extinct in the State of New York, of a Master in Chancery, had been conferred upon me. It was not a very arduous office, but very pleasantly remunerative. I seldom lose my temper; much more seldom indulge in dangerous indignation at wrongs and outrages; but I must be permitted to be rash here and declare, that I consider the sudden and violent abrogation of the office of Master of Chancery, by the new Constitution, as a----premature act; inasmuch as I had counted upon a life-lease of the profits, whereas I only received those of a few short years. But this is by the way.

      My chambers were up stairs at No.--Wall-street. At one end they looked upon the white wall of the interior of a spacious sky-light shaft, penetrating the building from top to bottom. This view might have been considered rather tame than otherwise, deficient in what landscape painters call "life." But if so, the view from the other end of my chambers offered, at least, a contrast, if nothing more. In that direction my windows commanded an unobstructed view of a lofty brick wall,black by age and everlasting shade; which wall required no spy-glass to bring out its lurking beauties, but for the benefit of all near-sighted spectators, was pushed up to within ten feet of my window panes. Owing to the great height of the surrounding buildings, and my chambers being on the second floor, the interval between this wall and mine not a little resembled a huge square cistern.

      At the period just preceding the advent of Bartleby, I had two persons as copyists in my employment, and a promising lad as an office-boy. First, Turkey; second, Nippers; third, Ginger Nut.These may seem names, the like of which are not usually found in the Directory. In truth they were nicknames, mutually conferred upon each other by my three clerks, and were deemed expressive of their respective persons or characters. Turkey was a short, pursy Englishman of about my own age, that is, somewhere not far from sixty. In the morning, one might say, his face was of a fine florid hue, but after twelve o'clock, meridian-- his dinner hour-- it blazed like a grate full of Christmas coals; and continued blazing--but, as it were, with a gradual wane--till 6 o'clock, P.M. or thereabouts, after which I saw no more of the proprietor of the face, which gaining its meridian with the sun, seemed to set with it, to rise, culminate, and decline the following day, with the like regularity and undiminished glory. There are many singular coincidences I have known in the course of my life, not the least among which was the fact that exactly when Turkey displayed his fullest beams from his red and radiant countenance, just then, too, at the critical moment, began the daily period when I considered his business capacities as seriously disturbed for the remainder of the twenty-four hours. Not that he was absolutely idle, or averse to business then; far from it. The difficulty was, he was apt to be altogether too energetic. There was a strange, inflamed, flurried, flighty recklessness of activity about him. He would be incautious in dipping his pen into his inkstand. All his blots upon my documents, were dropped there after twelve o'clock, meridian. Indeed, not only would he be reckless and sadly given to making blots in the afternoon, but some days he went further, and was rather noisy. At such times, too, his face flamed with augmented blazonry, as if cannel coal had been heaped on anthracite. He made an unpleasant racket with his chair; spilled his sand-box; in mending his pens, impatiently split them all to pieces, and threw them on the floor in a sudden passion; stood up and leaned over his table, boxing his papers about in a most indecorous manner, very sad to behold in an elderly manlike him. Nevertheless, as he was in many ways a most valuable person to me, and all the time before twelve o'clock, meridian, was the quickest, steadiest creature too, accomplishing a great deal of work in a style not easy to be matched--for these reasons, I was willingto overlook his eccentricities, though indeed, occasionally, I remonstrated with him. I did this very gently, however, because, though the civilest, nay, the blandest and most reverential of men in the morning, yet in the afternoon he was disposed, upon provocation, to be slightly rash with his tongue, in fact, insolent. Now, valuing his morning services as I did, and resolved not to lose them; yet, at the same time made uncomfortable by his inflamed ways after twelve o'clock; and being a man of peace, unwilling by my admonitions to call forth unseemingly retorts from him; I took upon me, one Saturday noon (he was always worse on Saturdays), to hint to him, very kindly, that perhaps now that he was growing old, it might be well to abridge his labors; in short, he need not come to my chambers after twelve o'clock, but, dinner over, had best go home to his lodgings and rest himself till tea-time. But no; he insisted upon his afternoon devotions. His countenance became intolerably fervid, as he oratorically assured me--gesticulating with a long ruler at the other end of the room--that if his services in the morning were useful, how indispensible, then, in the afternoon?

      "With submission, sir," said Turkey on this occasion, "I consider myself your right-hand man. In the morning I but marshal and deploy my columns; but in the afternoon I put myself at their head, and gallantly charge the foe, thus!"--and he made a violent thrust with the ruler.

      "But the blots, Turkey," intimated I.

      "True,--but, with submission, sir, behold these hairs! I am getting old. Surely, sir, a blot or two of a warm afternoon is not the page--is honorable. With submission, sir, we both are getting old."

      This appeal to my fellow-feeling was hardly to be resisted. At all events, I saw that go he would not. So I made up my mind to let him stay, resolving, nevertheless, to see to it, that during the afternoon he had to do with my less important papers.

      Nippers, the second on my list, was a whiskered, sallow, and, upon the whole, rather piratical-looking young man of about five and twenty. I always deemed him the victim of two evil powers-- ambition and indigestion. The ambition was evinced by a certain impatience of the duties of a mere copyist, an unwarrantable usurpation of strictly profession affairs, such as the original drawing up of legal documents. The indigestion seemed betokened in an occasional nervous testiness and grinning irritability, causing the teeth to audibly grind together over mistakes committed in copying; unnecessary maledictions, hissed, rather than spoken, in the heat of business; and especially by a continual discontent with the height of the table where he worked. Though of a very ingenious mechanical turn, Nippers could never get this table to suit him. He put chips under it, blocks of various sorts, bits of pasteboard, and at last went so far as to attempt an exquisite adjustment by final pieces of folded blotting-paper. But no invention would answer. If, for the sake of easing his back, he brought the table lid at a sharp angle well up towards his chin, and wrote there like a man using the steep roof of a Dutch house for his desk:--then he declared that it stopped the circulation in his arms. If now he lowered the table to his waistbands, and stooped over it in writing, then there was a sore aching in his back. In short, the truth of the matter was, Nippers knew not what he wanted. Or, if he wanted anything, it was to be rid of a scrivener's table altogether. Among the manifestations of his diseased ambition was a fondness he had for receiving visits from certain ambiguous-looking fellows in seedy coats, whom he called his clients. Indeed I was aware that not only was he, at times, considerable of a ward-politician, but he occasionally did a little businessat the Justices' courts, and was not unknown on the steps of the Tombs. I have good reason to believe, however, that one individual who called upon him at my chambers, and who, with a grand air, he insisted was his client, was no other than a dun, and the alleged title-deed, a bill. But with all his failings, and the annoyances he caused me, Nippers, like his compatriot Turkey, was a very useful man to me; wrote a neat, swift hand; and, when he chose, was not deficient in a gentlemanly sort of deportment. Added to this, he always dressedin a gentlemanly sort of way; and so, incidentally, reflected credit upon my chambers. Whereas with respect to Turkey, I had much ado to keep him from being a reproach to me. His clothes were apt to look oily and smell of eating-houses. He wore his pantaloons very loose and baggy in summer. His coats were execrable; his hat not to be handled. But while the hat was a thing of indifference to me, inasmuch as his natural civility and deference, as a dependent Englishman, always led him to doff it the moment he entered the room, yet his coat was another matter. Concerning his coats, I reasoned with him; but with no effect. The truth was, I suppose, that a man with so small an income, could not afford to sport such a lustrous face and a lustrous coat at one and the same time. As Nippers once observed, Turkey's money went chiefly for red ink. One winter day I presented Turkey with a highly-respectable looking coat of my own, a padded gray coat, of a most comfortable warmth, and which buttoned straight up from the knee to the neck. I thought Turkey would appreciate the favor, and abate his rashness and obstreperousness of afternoons. But no. I verily believe that buttoning himself up in so downy and blanket-like a coat had a pernicious effect upon him; upon the same principle that too much oats are bad for horses. In fact, precisely as a rash, restive horse is said to feel his oats, so Turkey felt his coat. It made him insolent. He was a man whom prosperity harmed.

      Though concerning the self-indulgent habits of Turkey I had my own private surmises, yet touching Nippers I was well persuaded that whatever might be his faults in other respects, he was, at least, a temperate young man. But indeed, nature herself seemed to have been his vintner, and at his birth charged him so thoroughly with an irritable, brandy-like disposition, that all subsequent potations were needless. When I consider how, amid the stillness of my chambers, Nippers would sometimes impatiently rise from his seat, and stooping over his table, spread his arms wide apart, seize the whole desk, and move it, and jerk it, with a grim, grinding motion on the floor, as if the table were a perverse voluntary agent, intent on thwarting and vexing him; I plainly perceive that for Nippers, brandy and water were altogether superfluous.

      It was fortunate for me that, owing to its course--indigestion--the irritability and consequent nervousness of Nippers, were mainly observable in the morning, while in the afternoon he was comparatively mild. So that Turkey's paroxysms only coming on about twelve o'clock, I never had to do with their eccentricities at one time. Their fits relieved each other like guards. When Nippers' was on, Turkey's was off, and vice versa. This was a good natural arrangement under the circumstances.

      Ginger Nut, the third on my list, was a lad some twelve years old. His father was a carman, ambitious of seeing his son on the bench instead of a cart, before he died. So he sent him to my office as a student at law, errand boy, and cleaner and sweeper, at the rate of one dollar a week. He had a little desk to himself, but he did not use it much. Upon inspection, the drawer exhibited a great array of the shells of various sorts of nuts. Indeed, to this quick-witted youth the whole noble science of the law was contained in a nut-shell. Not the least among the employments of Ginger Nut, as well as one which he discharged with the most alacrity, was his duty as cake and apple purveyor for Turkey and Nippers. Copying law papers being proverbially a dry, husky sort of business, my two scriveners were fain to moisten their mouths very often with Spitzenbergs to be had at the numerous stalls nigh the Custom House and Post Office. Also, they sent Ginger Nut very frequently for that peculiar cake--small, flat, round, and very spicy--after which he had been named by them. Of a cold morning when business was but dull, Turkey would gobble up scores of these cakes, as if they were mere wafers--indeed they sell them at the rate of six or eight for a penny--the scrape of his pen blending with the crunching of the crisp particles in his mouth. Of all the fiery afternoon blunders and flurried rashnesses of Turkey, was his once moistening a ginger-cake between his lips, and clapping it on to a mortgage for a seal. I came within an ace of dismissing him then. But he mollified me by making an oriental bow, and saying--"With submission, sir, it was generous of me to find you in stationery on my own account."

      Now my original business--that of a conveyancer and title hunter, and drawer-up of recondite documents of all sorts--was considerably increased by receiving the master's office. There was now great work for scriveners. Not only must I push the clerks already with me, but I must have additional help. In answer to my advertisement, a motionless young man one morning, stood upon my office threshold, the door being open, for it was summer. I can see that figure now--pallidly neat, pitiably respectable, incurably forlorn! It was Bartleby.

      After a few words touching his qualifications, I engaged him, glad to have among my corps of copyists a man of so singularly sedate an aspect, which I thought might operate beneficially upon the flighty temper of Turkey, and the fiery one of Nippers.

      I should have stated before that ground glass folding-doors divided my premises into two parts, one of which was occupied by my scriveners, the other by myself. According to my humor I threw open these doors, or closed them. I resolved to assign Bartleby a corner by the folding-doors, but on my side of them, so as to have this quiet man within easy call, in case any trifling thing was to be done. I placed his desk close up to a small side window in that part of the room, a window which originally had afforded a lateral view of certain grimy back-yards and bricks, but which, owing to subsequent erections, commanded at present no view at all, though it gave some light. Within three feet of the panes was a wall, and the light came down from far above, between two lofty buildings, as from a very small opening in a dome. Still further to a satisfactory arrangement, I procured a high green folding screen, which might entirely isolate Bartleby from my sight, though not remove him from my voice. And thus, in a manner, privacy and society were conjoined.

      At first Bartleby did an extraordinary quantity of writing. As if long famishingfor something to copy, he seemed to gorge himself on my documents. There was no pause for digestion. He ran a day and night line, copying by sun-light and by candle-light. I should have been quite delighted with his application, had be been cheerfully industrious. But he wrote on silently, palely, mechanically.

      It is, of course, an indispensable part of a scrivener's business to verify the accuracy of his copy, word by word. Where there are two or more scriveners in an office, they assist each other in this examination, one reading from the copy, the other holding the original. It is a very dull, wearisome, and lethargic affair. I can readily imagine that to some sanguine temperaments it would be altogether intolerable. For example, I cannot credit that the mettlesome poet Byron would have contentedly sat down with Bartleby to examine a law document of, say five hundred pages, closely written in a crimpy hand.

      Now and then, in the haste of business, it had been my habit to assist in comparing some brief document myself, calling Turkey or Nippers for this purpose. One object I had in placing Bartleby so handy to me behind the screen, was to avail myself of his services on such trivial occasions. It was on the third day, I think, of his being with me, and before any necessity had arisen for having his own writing examined, that, being much hurried to complete a small affair I had in hand, I abruptly called to Bartleby. In my haste and natural expectancy of instant compliance, I sat with my head bent over the original on my desk, and my right hand sideways, and somewhat nervously extended with the copy, so that immediately upon emerging from his retreat, Bartleby might snatch it and proceed to business without the least delay.

      In this very attitude did I sit when I called to him, rapidly stating what it was I wanted him to do--namely, to examine a small paper with me. Imagine my surprise, nay, my consternation, when without moving from his privacy, Bartleby in a singularly mild, firm voice, replied,"I would prefer not to."

      I sat awhile in perfect silence, rallying my stunned faculties. Immediately it occurred to me that my ears had deceived me, or Bartleby had entirely misunderstood my meaning. I repeated my request in the clearest tone I could assume. But in quite as clear a one came the previous reply, "I would prefer not to."

      "Prefer not to," echoed I, rising in high excitement, and crossing the room with a stride, "What do you mean? Are you moon-struck? I want you to help me compare this sheet here--take it," and I thrust it towards him.

      "I would prefer not to," said he.

      I looked at him steadfastly. His face was leanly composed; his gray eye dimly calm. Not a wrinkle of agitation rippled him. Had there been the least uneasiness, anger, impatience or impertinence in his manner; in other words, had there been any thing ordinarily human about him, doubtless I should have violently dismissed him from the premises. But as it was, I should have as soon thought of turning my pale plaster-of-paris bust of Cicero out of doors. I stood gazing at him awhile, as he went on with his own writing, and then reseated myself at my desk. This is very strange, thought I. What had one best do? But my business hurried me. I concluded to forget the matter for the present, reserving it for my future leisure. So calling Nippers from the other room, the paper was speedily examined.

      A few days after this, Bartleby concluded four lengthy documents, being quadruplicates of a week's testimony taken before me in my High Court of Chancery. It became necessary to examine them. It was an important suit, and great accuracy was imperative. Having all things arranged I called Turkey, Nippers and Ginger Nut from the next room, meaning to place the four copies in the hands of my four clerks, while I should read from the original. Accordingly Turkey, Nippers and Ginger Nut had taken their seats in a row, each with his document in hand, when I called to Bartleby to join this interesting group.

      "Bartleby! quick, I am waiting."

      I heard a low scrape of his chair legs on the unscraped floor, and soon he appeared standing at the entrance of his hermitage.

      "What is wanted?" said he mildly.

      "The copies, the copies," said I hurriedly. "We are going to examine them. There"--and I held towards him the fourth quadruplicate.

      "I would prefer not to," he said, and gently disappeared behind the screen.

      For a few moments I was turned into a pillar of salt, standing at the head of my seated column of clerks. Recovering myself, I advanced towards the screen, and demanded the reason for such extraordinary conduct.

      "Why do you refuse?"

      "I would prefer not to."

      With any other man I should have flown outright into a dreadful passion, scorned all further words, and thrust him ignominiously from my presence. But there was something about Bartleby that not only strangely disarmed me, but in a wonderful manner touched and disconcerted me. I began to reason with him.

      "These are your own copies we are about to examine. It is labor saving to you, because one examination will answer for your four papers. It is common usage. Every copyist is bound to help examine his copy. Is it not so? Will you not speak? Answer!"

      "I prefer not to," he replied in a flute-like tone. It seemed to me that while I had been addressing him, he carefully revolved every statement that I made; fully comprehended the meaning; could not gainsay the irresistible conclusion; but, at the same time, some paramount consideration prevailed with him to reply as he did.

      "You are decided, then, not to comply with my request--a request made according to common usage and common sense?"

      He briefly gave me to understand that on that point my judgment was sound. Yes: his decision was irreversible.

      It is not seldom the case that when a man is browbeaten in some unprecedented and violently unreasonable way, he begins to stagger in his own plainest faith. He begins, as it were, vaguely to surmise that, wonderful as it may be, all the justice and all the reason is on the other side. Accordingly, if any disinterested persons are present, he turns to them for some reinforcement for his own faltering mind.

      "Turkey," said I, "what do you think of this? Am I not right?"

      "With submission, sir," said Turkey, with his blandest tone, "I think that you are."

      "Nippers," said I, "what do you think of it?"

      "I think I should kick him out of the office."

      (The reader of nice perceptions will here perceive that, it being morning, Turkey's answer is couched in polite and tranquil terms, but Nippers replies in ill-tempered ones. Or, to repeat a previous sentence, Nipper's ugly mood was on duty, and Turkey's off.)

      "Ginger Nut," said I, willing to enlist the smallest suffrage in my behalf, "what do you think of it?"

      "I think, sir, he's a little luny," replied Ginger Nut, with a grin.

      "You hear what they say," said I, turning towards the screen, "come forth and do your duty."

      But he vouchsafed no reply. I pondered a moment in sore perplexity. But once more business hurried me. I determined again to postpone the consideration of this dilemma to my future leisure. With a little trouble we made out to examine the papers without Bartleby, though at every page or two, Turkey deferentially dropped his opinion that this proceeding was quite out of the common; while Nippers, twitching in his chair with a dyspeptic nervousness, ground out between his set teeth occasional hissing maledictions against the stubborn oaf behind the screen. And for his (Nipper's) part, this was the first and the last time he would do another man's business without pay.

      Meanwhile Bartleby sat in his hermitage, oblivious to every thing but his own peculiar business there.

      Some days passed, the scrivener being employed upon another lengthy work. His late remarkable conduct led me to regard his way narrowly. I observed that he never went to dinner; indeed that he never went any where. As yet I had never of my personal knowledge known him to be outside of my office. He was a perpetual sentry in the corner. At about eleven o'clock though, in the morning, I noticed that Ginger Nut would advance toward the opening in Bartleby's screen, as if silently beckoned thither by a gesture invisible to me where I sat. That boy would then leave the office jingling a few pence, and reappear with a handful of ginger-nuts which he delivered in the hermitage, receiving two of the cakes for his trouble.

      He lives, then, on ginger-nuts, thought I; never eats a dinner, properly speaking; he must be a vegetarian then, but no; he never eats even vegetables, he eats nothing but ginger-nuts. My mind then ran on in reveries concerning the probable effects upon the human constitution of living entirely on ginger-nuts. Ginger-nuts are so called because they contain ginger as one of their peculiar constituents, and the final flavoring one. Now what was ginger? A hot, spicy thing. Was Bartleby hot and spicy? Not at all. Ginger, then, had no effect upon Bartleby. Probably he preferred it should have none.

      Nothing so aggravates an earnest person as a passive resistance. If the individual so resisted be of a not inhumane temper, and the resisting one perfectly harmless in his passivity; then, in the better moods of the former, he will endeavor charitably to construe to his imagination what proves impossible to be solved by his judgment. Even so, for the most part, I regarded Bartleby and his ways. Poor fellow! thought I, he means no mischief; it is plain he intends no insolence; his aspect sufficiently evinces that his eccentricities are involuntary. He is useful to me. I can get along with him. If I turn him away, the chances are he will fall in with some less indulgent employer, and then he will be rudely treated, and perhaps driven forth miserably to starve. Yes. Here I can cheaply purchase a delicious self-approval. To befriend Bartleby; to humor him in his strange willfulness, will cost me little or nothing, while I lay up in my soul what will eventually prove a sweet morsel for my conscience. But this mood was not invariable with me. The passiveness of Bartleby sometimes irritated me. I felt strangely goaded on to encounter him in new opposition, to elicit some angry spark from him answerable to my own. But indeed I might as well have essayed to strike fire with my knuckles against a bit of Windsor soap. But one afternoon the evil impulse in me mastered me, and the following little scene ensued:

      "Bartleby," said I, "when those papers are all copied, I will compare them with you."

      "I would prefer not to."

      "How? Surely you do not mean to persist in that mulish vagary?"

      No answer.

      I threw open the folding-doors near by, and turning upon Turkey and Nippers, exclaimed in an excited manner--

      "He says, a second time, he won't examine his papers. What do you think of it, Turkey?"

      It was afternoon, be it remembered. Turkey sat glowing like a brass boiler, his bald head steaming, his hands reeling among his blotted papers.

      "Think of it?" roared Turkey; "I think I'll just step behind his screen, and black his eyes for him!"

      So saying, Turkey rose to his feet and threw his arms into a pugilistic position. He was hurrying away to make good his promise, when I detained him, alarmed at the effect of incautiously rousing Turkey's combativeness after dinner.

      "Sit down, Turkey," said I, "and hear what Nippers has to say. What do you think of it, Nippers? Would I not be justified in immediately dismissing Bartleby?"

      "Excuse me, that is for you to decide, sir. I think his conduct quite unusual, and indeed unjust, as regards Turkey and myself. But it may only be a passing whim."

      "Ah," exclaimed I, "you have strangely changed your mind then--you speak very gently of him now."

      "All beer," cried Turkey; "gentleness is effects of beer--Nippers and I dined together to-day. You see how gentle I am, sir. Shall I go and black his eyes?"

      "You refer to Bartleby, I suppose. No, not to-day, Turkey," I replied; "pray, put up your fists."

      I closed the doors, and again advanced towards Bartleby. I felt additional incentives tempting me to my fate. I burned to be rebelled against again. I remembered that Bartleby never left the office.

      "Bartleby," said I, "Ginger Nut is away; just step round to the Post Office, won't you? (it was but a three minutes walk,) and see if there is any thing for me."

      "I would prefer not to."

      "You will not?"

      "I prefer not."

      I staggered to my desk, and sat there in a deep study. My blind inveteracy returned. Was there any other thing in which I could procure myself to be ignominiously repulsed by this lean, penniless with?--my hired clerk? What added thing is there, perfectly reasonable, that he will be sure to refuse to do?

      "Bartleby!"

      No answer.

      "Bartleby," in a louder tone.

      No answer.

      "Bartleby," I roared.

      Like a very ghost, agreeably to the laws of magical invocation, at the third summons, he appeared at the entrance of his hermitage.

      "Go to the next room, and tell Nippers to come to me."

      "I prefer not to," he respectfully and slowly said, and mildly disappeared.

      "Very good, Bartleby," said I, in a quiet sort of serenely severe self-possessed tone, intimating the unalterable purpose of some terrible retribution very close at hand. At the moment I half intended something of the kind. But upon the whole, as it was drawing towards my dinner-hour, I thought it best to put on my hat and walk home for the day, suffering much from perplexity and distress of mind.

      Shall I acknowledge it? The conclusion of this whole business was that it soon became a fixed fact of my chambers, that a pale young scrivener, by the name of Bartleby, had a desk there; that he copied for me at the usual rate of four cents a folio (one hundred words); but he was permanently exempt from examining the work done by him, that duty being transferred to Turkey and Nippers, one of compliment doubtless to their superior acuteness; moreover, said Bartleby was never on any account to be dispatched on the most trivial errand of any sort; and that even if entreated to take upon him such a matter, it was generally understood that he would prefer not to--in other words, that he would refuse point-blank.

      32 As days passed on, I became considerably reconciled to Bartleby. His steadiness, his freedom from all dissipation, his incessant industry (except when he chose to throw himself into a standing revery behind his screen), his great stillness, his unalterableness of demeanor under all circumstances, made him a valuable acquisition. One prime thing was this,--he was always there;--first in the morning, continually through the day, and the last at night. I had a singular confidence in his honesty. I felt my most precious papers perfectly safe in his hands. Sometimes to be sure I could not, for the very soul of me, avoid falling into sudden spasmodic passions with him. For it was exceeding difficult to bear in mind all the time those strange peculiarities, privileges, and unheard of exemptions, forming the tacit stipulations on Bartleby's part under which he remained in my office. Now and then, in the eagerness of dispatching pressing business, I would inadvertently summon Bartleby, in a short, rapid tone, to put his finger, say, on the incipient tie of a bit of red tape with which I was about compressing some papers. Of course, from behind the screen the usual answer, "I prefer not to," was sure to come; and then, how could a human creature with the common infirmities of our nature, refrain from bitterly exclaiming upon such perverseness--such unreasonableness. However, every added repulse of this sort which I received only tended to lessen the probability of my repeating the inadvertence.

      Here is must be said, that according to the custom of most legal gentlemen occupying chambers in densely-populated law buildings, there were several keys to my door. One was kept by a woman residing in the attic, which person weekly scrubbed and daily swept and dusted my apartments. Another was kept by Turkey for convenience sake. The third I sometimes carried in my own pocket. The fourth I knew not who had.

      Now, one Sunday morning I happened to go to Trinity Church, to hear a celebrated preacher, and finding myself rather early on the ground, I thought I would walk round to my chambers for a while. Luckily I had my key with me; but upon applying it to the lock, I found it resisted by something inserted from the inside. Quite surprised, I called out; when to my consternation a key was turned from within; and thrusting his lean visage at me, and holding the door ajar, the apparition of Bartleby appeared, in his shirt sleeves, and otherwise in a strangely tattered dishabille, saying quietly that he was sorry, but he was deeply engaged just then, and--preferred not admitting me at present. In a brief word or two, he moreover added, that perhaps I had better walk round the block two or three times, and by that time he would probably have concluded his affairs. Now, the utterly unsurmised appearance of Bartleby, tenanting my law-chambers of a Sunday morning, with his cadaverously gentlemanly nonchalance, yet withal firm and self-possessed, had such a strange effect upon me, that incontinently I slunk away from my own door, and did as desired. But not without sundry twinges of impotent rebellion against the mild effrontery of this unaccountable scrivener. Indeed, it was his wonderful mildness chiefly, which not only disarmed me, but unmanned me, as it were. For I consider that one, for the time, is a sort of unmanned when he tranquilly permits his hired clerk to dictate to him, and order him away from his own premises. Furthermore, I was full of uneasiness as to what Bartleby could possibly be doing in my office in his shirt sleeves, and in an otherwise dismantled condition of a Sunday morning. Was any thing amiss going on? Nay, that was out of the question. It was not to be thought of for a moment that Bartleby was an immoral person. But what could he be doing there?--copying? Nay again, whatever might be his eccentricities, Bartleby was an eminently decorous person. He would be the last man to sit down to his desk in any state approaching to nudity. Besides, it was Sunday; and there was something about Bartleby that forbade the supposition that we would by any secular occupation violate the proprieties of the day.

      Nevertheless, my mind was not pacified; and full of a restless curiosity, at last I returned to the door. Without hindrance I inserted my key, opened it, and entered. Bartleby was not to be seen. I looked round anxiously, peeped behind his screen; but it was very plain that he was gone. Upon more closely examining the place, I surmised that for an indefinite period Bartleby must have ate, dressed, and slept in my office, and that too without plate, mirror, or bed. The cushioned seat of a rickety old sofa in one corner bore t faint impress of a lean, reclining form. Rolled away under his desk, I found a blanket; under the empty grate, a blacking box and brush; on a chair, a tin basin, with soap and a ragged towel; in a newspaper a few crumbs of ginger-nuts and a morsel of cheese. Yet, thought I, it is evident enough that Bartleby has been making his home here, keeping bachelor's hallall by himself. Immediately then the thought came sweeping across me, What miserable friendlessness and loneliness are here revealed! His poverty is great; but his solitude, how horrible! Think of it. Of a Sunday, Wall-street is deserted as Petra; and every night of every day it is an emptiness. This building too, which of week-days hums with industry and life, at nightfall echoes with sheer vacancy, and all through Sunday is forlorn. And here Bartleby makes his home; sole spectator of a solitude which he has seen all populous--a sort of innocent and transformed Marius brooding among the ruins of Carthage!

      For the first time in my life a feeling of overpowering stinging melancholy seized me. Before, I had never experienced aught but a not-unpleasing sadness. The bond of a common humanity now drew me irresistibly to gloom. A fraternal melancholy! For both I and Bartleby were sons of Adam. I remembered the bright silks and sparkling faces I had seen that day in gala trim, swan-like sailing down the Mississippi of Broadway; and I contrasted them with the pallid copyist, and thought to myself, Ah, happiness courts the light, so we deem the world is gay; but misery hides aloof, so we deem that misery there is none. These sad fancyings-- chimeras, doubtless, of a sick and silly brain--led on to other and more special thoughts, concerning the eccentricities of Bartleby. Presentiments of strange discoveries hovered round me. The scrivener's pale form appeared to me laid out, among uncaring strangers, in its shivering winding sheet.

      Suddenly I was attracted by Bartleby's closed desk, the key in open sight left in the lock.

      I mean no mischief, seek the gratification of no heartless curiosity, thought I; besides, the desk is mine, and its contents too, so I will make bold to look within. Every thing was methodically arranged, the papers smoothly placed. The pigeon holes were deep, and removing the files of documents, I groped into their recesses. Presently I felt something there, and dragged it out. It was an old bandanna handkerchief, heavy and knotted. I opened it, and saw it was a savings' bank.

      I now recalled all the quiet mysteries which I had noted in the man. I remembered that he never spoke but to answer; that though at intervals he had considerable time to himself, yet I had never seen him reading--no, not even a newspaper; that for long periods he would stand looking out, at his pale window behind the screen, upon the dead brick wall; I was quite sure he never visited any refectory or eating house; while his pale face clearly indicated that he never drank beer like Turkey, or tea and coffee even, like other men; that he never went any where in particular that I could learn; never went out for a walk, unless indeed that was the case at present; that he had declined telling who he was, or whence he came, or whether he had any relatives in the world; that though so thin and pale, he never complained of ill health. And more than all, I remembered a certain unconscious air of pallid--how shall I call it?--of pallid haughtiness, say, or rather an austere reserve about him, which had positively awed me into my tame compliance with his eccentricities, when I had feared to ask him to do the slightest incidental thing for me, even though I might know, from his long-continued motionlessness, that behind his screen he must be standing in one of those dead-wall reveries of his.

      Revolving all these things, and coupling them with the recently discovered fact that he made my office his constant abiding place and home, and not forgetful of his morbid moodiness; revolving all these things, a prudential feeling began to steal over me. My first emotions had been those of pure melancholy and sincerest pity; but just in proportion as the forlornness of Bartleby grew and grew to my imagination, did that same melancholy merge into fear, that pity into repulsion. So true it is, and so terrible too, that up to a certain point the thought or sight of misery enlists our best affections; but, in certain special cases, beyond that point it does not. They err who would assert that invariably this is owing to the inherent selfishness of the human heart. It rather proceeds from a certain hopelessness of remedying excessive and organic ill. To a sensitive being, pity is not seldom pain. And when at last it is perceived that such pity cannot lead to effectual succor, common sense bids the soul be rid of it. What I saw that morning persuaded me that the scrivener was the victim of innate and incurable disorder. I might give alms to his body; but his body did not pain him; it was his soul that suffered, and his soul I could not reach.

      I did not accomplish the purpose of going to Trinity Church that morning. Somehow, the things I had seen disqualified me for the time from church-going. I walked homeward, thinking what I would do with Bartleby. Finally, I resolvedupon this;--I would put certain calm questions to him the next morning, touching his history, &c., and if he declined to answer then openly and reservedly (and I supposed he would prefer not), then to give him a twenty dollar bill over and above whatever I might owe him, and tell him his services were no longer required; but that if in any other way I could assist him, I would be happy to do so, especially if he desired to return to his native place, wherever that might be, I would willingly help to defray the expenses. Moreover, if after reaching home, he found himself at any time in want of aid, a letter from him would be sure of a reply.

      The next morning came.

      "Bartleby," said I, gently calling to him behind the screen.

      No reply.

      "Bartleby," said I, in a still gentler tone, "come here; I am not going to ask you to do any thing you would prefer not to do--I simply wish to speak to you."

      Upon this he noiselessly slid into view.

      "Will you tell me, Bartleby, where you were born?"

      "I would prefer not to."

      "Will you tell me anything about yourself?"

      "I would prefer not to."

      "But what reasonable objection can you have to speak to me? I feel friendly towards you."

      He did not look at me while I spoke, but kept his glance fixed upon my bust of Cicero, which as I then sat, was directly behind me, some six inches above my head. "What is your answer, Bartleby?" said I, after waiting a considerable time for a reply, during which his countenance remained immovable, only there was the faintest conceivable tremor of the white attenuated mouth.

      "At present I prefer to give no answer," he said, and retired into his hermitage.

      It was rather weak in me I confess, but his manner on this occasion nettled me. Not only did there seem to lurk in it a certain disdain, but his perverseness seemed ungrateful, considering the undeniable good usage and indulgence he had received from me.

      Again I sat ruminating what I should do.Mortified as I was at his behavior, and resolved as I had been to dismiss him when I entered my office, nevertheless I strangely felt something superstitious knocking at my heart, and forbidding me to carry out my purpose, and denouncing me for a villain if I dared to breathe one bitter word against this forlornest of mankind. At last, familiarly drawing my chair behind his screen, I sat down and said: "Bartleby, never mind then about revealing your history; but let me entreat you, as a friend, to comply as far as may be with the usages of this office. Say now you will help to examine papers tomorrow or next day: in short, say now that in a day or two you will begin to be a little reasonable:--say so, Bartleby."

      "At present I would prefer not to be a little reasonable was his idly cadaverous reply.,"

      Just then the folding-doors opened, and Nippers approached. He seemed suffering from an unusually bad night's rest, induced by severer indigestion than common. He overheard those final words of Bartleby.

      "Prefer not, eh?" gritted Nippers--"I'd prefer him, if I were you, sir," addressing me--"I'd prefer him; I'd give him preferences, the stubborn mule! What is it, sir, pray, that he prefers not to do now?"

      Bartleby moved not a limb.

      "Mr. Nippers," said I, "I'd prefer that you would withdraw for the present."

      Somehow, of late I had got into the way of involuntary using this word "prefer" upon all sorts of not exactly suitable occasions. And I trembled to think that my contact with the scrivener had already and seriously affected me in a mental way. And what further and deeper aberration might it not yet produce? This apprehension had not been without efficacy in determining me to summary means.

      As Nippers, looking very sour and sulky, was departing, Turkey blandly and deferentially approached.

      "With submission, sir," said he, "yesterday I was thinking about Bartleby here, and I think that if he would but prefer to take a quart of good ale every day, it would do much towards mending him, and enabling him to assist in examining his papers."

      "So you have got the word too," said I, slightly excited.

      "With submission, what word, sir," asked Turkey, respectfully crowding himself into the contracted space behind the screen, and by so doing, making me jostle the scrivener. "What word, sir?"

      "I would prefer to be left alone here," said Bartleby, as if offended at being mobbed in his privacy.

      "That's the word, Turkey," said I--"that's it."

      "Oh, prefer oh yes--queer word. I never use it myself. But, sir as I was saying, if he would but prefer--"

      "Turkey," interrupted I, "you will please withdraw."

      "Oh, certainly, sir, if you prefer that I should."

      As he opened the folding-door to retire, Nippers at his desk caught a glimpse of me, and asked whether I would prefer to have a certain paper copied on blue paper or white. He did not in the least roguishly accent the word prefer. It was plain that it involuntarily rolled from his tongue. I thought to myself, surely I must get rid of a demented man, who already has in some degree turned the tongues, if not the heads of myself and clerks. But I thought it prudent not to break the dismission at once.

      The next day I noticed that Bartleby did nothing but stand at his window in his dead-wall revery. Upon asking him why he did not write, he said that he had decided upon doing no more writing.

      "Why, how now? what next?" exclaimed I, "do no more writing?"

      "No more."

      "And what is the reason?"

      "Do you not see the reason for yourself," he indifferently replied.

      I looked steadfastly at him, and perceived that his eyes looked dull and glazed. Instantly it occurred to me, that his unexampled diligence in copying by his dim window for the first few weeks of his stay with me might have temporarily impaired his vision.

      I was touched. I said something in condolence with him. I hinted that of course he did wisely in abstaining from writing for a while; and urged him to embrace that opportunity of taking wholesome exercise in the open air. This, however, he did not do. A few days after this, my other clerks being absent, and being in a great hurry to dispatch certain letters by the mail, I thought that, having nothing else earthly to do, Bartleby would surely be less inflexible than usual, and carry these letters to the post-office. But he blankly declined. So, much to my inconvenience, I went myself.

      Still added days went by. Whether Bartleby's eyes improved or not, I could not say. To all appearance, I thought they did. But when I asked him if they did, he vouchsafed no answer. At all events, he would do no copying. At last, in reply to my urgings, he informed me that he had permanently given up copying.

      "What!" exclaimed I; "suppose your eyes should get entirely well- better than ever before--would you not copy then?"

      "I have given up copying," he answered, and slid aside.

      He remained as ever, a fixture in my chamber. Nay--if that were possible--he became still more of a fixture than before. What was to be done? He would do nothing in the office: why should he stay there? In plain fact, he had now become a millstone to me, not only useless as a necklace, but afflictive to bear. Yet I was sorry for him. I speak less than truth when I say that, on his own account, he occasioned me uneasiness. If he would but have named a single relative or friend, I would instantly have written, and urged their taking the poor fellow away to some convenient retreat. But he seemed alone, absolutely alone in the universe. A bit of wreck</font> in the mid Atlantic. At length, necessities connected with my business tyrannized over all other considerations. Decently as I could, I told Bartleby that in six days' time he must unconditionally leave the office. I warned him to take measures, in the interval, for procuring some other abode. I offered to assist him in this endeavor, if he himself would but take the first step towards a removal. "And when you finally quit me, Bartleby," added I, "I shall see that you go not away entirely unprovided. Six days from this hour, remember."

      At the expiration of that period, I peeped behind the screen, and lo! Bartleby was there.

      I buttoned up my coat, balanced myself; advanced slowly towards him, touched his shoulder, and said, "The time has come; you must quit this place; I am sorry for you; here is money; but you must go."

      "I would prefer not," he replied, with his back still towards me.

      "You must."

      He remained silent.

      Now I had an unbounded confidence in this man's common honesty. He had frequently restored to me six pences and shillings carelessly dropped upon the floor, for I am apt to be very reckless in such shirt-button affairs. The proceeding then which followed will not be deemed extraordinary. "Bartleby," said I, "I owe you twelve dollars on account; here are thirty-two; the odd twenty are yours.--Will you take it? and I handed the bills towards him.

      But he made no motion.

      "I will leave them here then," putting them under a weight on the table. Then taking my hat and cane and going to the door I tranquilly turned and added--"After you have removed your things from these offices, Bartleby, you will of course lock the door--since every one is now gone for the day but you--and if you please, slip your key underneath the mat, so that I may have it in the morning. I shall not see you again; so good-bye to you. If hereafter in your new place of abode I can be of any service to you, do not fail to advise me by letter. Good-bye, Bartleby, and fare you well."

      But he answered not a word; like the last column of some ruined temple, he remained standing mute and solitary in the middle of the otherwise deserted room.

      As I walked home in a pensive mood, my vanity got the better of my pity. I could not but highly plume myself on my masterly management in getting rid of Bartleby. Masterly I call it, and such it must appear to any dispassionate thinker. The beauty of my procedure seemed to consist in its perfect quietness. There was no vulgar bullying, no bravado of any sort, no choleric hectoring and striding to and fro across the apartment, jerking out vehement commands for Bartleby to bundle himself off with his beggarly traps. Nothing of the kind. Without loudly bidding Bartleby depart--as an inferior genius might have done--I assumed the ground that depart he must; and upon the assumption built all I had to say. The more I thought over my procedure, the more I was charmed with it. Nevertheless, next morning, upon awakening, I had my doubts,--I had somehow slept off the fumes of vanity. One of the coolest and wisest hours a man has, is just after he awakes in the morning. My procedure seemed as sagacious as ever,--but only in theory. How it would prove in practice--there was the rub. It was truly a beautiful thought to have assumed Bartleby's departure; but, after all, that assumption was simply my own, and none of Bartleby's. The great point was, not whether I had assumed that he would quit me, but whether he would prefer so to do. He was more a man of preferences than assumptions.

      After breakfast, I walked down town, arguing the probabilities pro and con. One moment I thought it would prove a miserable failure, and Bartleby would be found all alive at my office as usual; the next moment it seemed certain that I should see his chair empty. And so I kept veering about. At the corner of Broadway and Canal- street, I saw quite an excited group of people standing in earnest conversation.

      "I'll take odds he doesn't," said a voice as I passed.

      "Doesn't go?--done!" said I, "put up your money."

      I was instinctively putting my hand in my pocket to produce my own, when I remembered that this was an election day. The words I had overheard bore no reference to Bartleby, but to the success or non-success of some candidate for the mayoralty. In my intent frame of mind, I had, as it were, imagined that all Broadway shared in my excitement, and were debating the same question with me. I passed on, very thankful that the uproar of the street screened my momentary absent-mindedness.

      As I had intended, I was earlier than usual at my office door. I stood listening for a moment. All was still. He must be gone. I tried the knob. The door was locked. Yes, my procedure had worked to a charm; he indeed must be vanished. Yet a certain melancholy mixed with this: I was almost sorry for my brilliant success. I was fumbling under the door mat for the key, which Bartleby was to have left there for me, when accidentally my knee knocked against a panel, producing a summoning sound, and in response a voice came to me from within--"Not yet; I am occupied."

      It was Bartleby.

      I was thunderstruck. For an instant I stood like the man who, pipe in mouth, was killed one cloudless afternoon long ago in Virginia, by summer lightning; at his own warm open window he was killed, and remained leaning out there upon the dreamy afternoon, till some one touched him, when he fell. "Not gone!" I murmured at last. But again obeying that wondrous ascendancy which the inscrutable scrivener had over me, and from which ascendancy, for all my chafing, I could not completely escape, I slowly went down stairs and out into the street, and while walking round the block, considered what I should next do in this unheard-of-perplexity. Turn the man out by an actual thrusting I could not; to drive him away by calling him hard names would not do; calling in the police was an unpleasant idea; and yet, permit him to enjoy his cadaverous triumph over me,--this too I could not think of. What was to be done? or, if nothing could be done, was there any thing further that I could assume in the matter? Yes, as before I had prospectively assumed that Bartleby would depart, so now I might retrospectively assume that departed he was. In the legitimate carrying out of this assumption, I might enter my office in a great hurry, and pretending not to see Bartleby at all, walk straight against him as if he were air. Such a proceeding would in a singular degree have the appearance of a home-thrust. It was hardly possible that Bartleby could withstand such an application of the doctrine of assumptions. But upon second thoughts the success of the plan seemed rather dubious. I resolved to argue the matter over with him again.

      Bartleby," said I, entering the office, with a quietly severe expression. "I am seriously displeased. I am pained, Bartleby. I had thought better of you. I had imagined you of such a gentlemanly organization, that in any delicate dilemma a slight hint would suffice--in short, an assumption. But it appears I am deceived. Why," I added, unaffectedly starting, "you have not even touched the money yet," pointing to it, just where I had left it the evening previous.

      He answered nothing.

      "Will you, or will you not, quit me?" I now demanded in a sudden passion, advancing close to him.

      "I would prefer not to quit you," he replied, gently emphasizing the not.

      "What earthly right have you to stay here? do you pay any rent? Do you pay my taxes? Or is this property yours?"

      He answered nothing.

      "Are you ready to go on and write now? Are your eyes recovered? Could you copy a small paper for me this morning? or help examine a few lines? or step round to the post-office? In a word, will you do any thing at all, to give a coloring to your refusal to depart the premises?"

      He silently retired into his hermitage.

      I was now in such a state of nervous resentment that I thought it but prudentto check myself at present from further demonstrations. Bartleby and I were alone. I remembered the tragedy of the unfortunate Adams and the still more unfortunate Colt in the solitary office of the latter; and how poor Colt, being dreadfully incensed by Adams, and imprudently permitting himself to get wildly excited, was at unawares hurried into his fatal act--an act which certainly no man could possibly deplore more than the actor himself. Often it had occurred to me in my ponderings upon the subject, that had that altercation taken place in the public street, or at a private residence, it would not have terminated as it did. It was the circumstance of being alone in a solitary office, up stairs, of a building entirely unhallowed by humanizing domestic associations--an uncarpeted office, doubtless of a dusty, haggard sort of appearance;--this it must have been, which greatly helped to enhance the irritable desperation of the hapless Colt.

      But when this old Adam of resentment rose in me and tempted me concerning Bartleby, I grappled him and threw him. How? Why, simply by recalling the divine injunction: "A new commandment give I unto you, that ye love one another." Yes, this it was that saved me. Aside from higher considerations, charity often operates as a vastly wise and prudent principle--a great safeguard to its possessor. Men have committed murder for jealousy's sake, and anger's sake, and hatred's sake, and selfishness' sake, and spiritual pride's sake; but no man that ever I heard of, ever committed a diabolical murder for sweet charity's sake. Mere self-interest, then, if no better motive can be enlisted, should, especially with high-tempered men, prompt all beings to charity and philanthropy. At any rate, upon the occasion in question, I strove to drown my exasperated feelings towards the scrivener by benevolently construing his conduct. Poor fellow, poor fellow! thought I, he don't mean any thing; and besides, he has seen hard times, and ought to be indulged.

      I endeavored also immediately to occupy myself, and at the same time to comfort my despondency.I tried to fancy that in the course of the morning, at such time as might prove agreeable to him, Bartleby, of his own free accord, would emerge from his hermitage, and take up some decided line of march in the direction of the door. But no. Half-past twelve o'clock came; Turkey began to glow in the face, overturn his inkstand, and become generally obstreperous; Nippers abated down into quietude and courtesy; Ginger Nut munched his noon apple; and Bartleby remained standing at his window in one of his profoundest deadwall reveries. Will it be credited? Ought I to acknowledge it? That afternoon I left the office without saying one further word to him.

      Some days now passed, during which, at leisure intervals I looked a little into Edwards on the Will," and "Priestly on Necessity." Under the circumstances, those books induced a salutary feeling. Gradually I slid into the persuasion that these troubles of mine touching the scrivener, had been all predestinated from eternity, and Bartleby was billeted upon me for some mysterious purpose of an all-wise Providence, which it was not for a mere mortal like me to fathom. Yes, Bartleby, stay there behind your screen, thought I; I shall persecute you no more; you are harmless and noiseless as any of these old chairs; in short, I never feel so private as when I know you are here. At least I see it, I feel it; I penetrate to the predestinated purpose of my life. I am content. Others may have loftier parts to enact; but my mission in this world, Bartleby, is to furnish you with office-room for such period as you may see fit to remain.

      I believe that this wise and blessed frame of mind would have continued with me, had it not been for the unsolicited and uncharitable remarks obtruded upon me by my professional friends who visited the rooms. But thus it often is, that the constant friction of illiberal minds wears out at last the best resolves of the more generous. Though to be sure, when I reflected upon it, it was not strange that people entering my office should be struck by the peculiar aspect of the unaccountable Bartleby, and so be tempted to throw out some sinister observations concerning him. Sometimes an attorney having business with me, and calling at my office, and finding no one but the scrivener there, would undertake to obtain some sort of precise information from him touching my whereabouts; but without heeding his idle talk, Bartleby would remain standing immovable in the middle of the room. So after contemplating him in that position for a time, the attorney would depart, no wiser than he came.

      Also, when a Reference was going on, and the room full of lawyers and witnesses and business was driving fast; some deeply occupied legal gentleman present, seeing Bartleby wholly unemployed, would request him to run round to his (the legal gentleman's) office and fetch some papers for him. Thereupon, Bartleby would tranquilly decline, and remain idle as before. Then the lawyer would give a great stare, and turn to me. And what could I say? At last I was made aware that all through the circle of my professional acquaintance, a whisper of wonder was running round, having reference to the strange creature I kept at my office. This worried me very much. And as the idea came upon me of his possibly turning out a long-lived man, and keep occupying my chambers, and denying my authority; and perplexing my visitors; and scandalizing my professional reputation; and casting a general gloom over the premises; keeping soul and body together to the last upon his savings (for doubtless he spent but half a dime a day), and in the end perhaps outlive me, and claim possession of my office by right of his perpetual occupancy: as all these dark anticipations crowded upon me more and more, and my friends continually intruded their relentless remarks upon the apparition in my room; a great change was wrought in me. I resolved to gather all my faculties together, and for ever rid me of this intolerable incubus.

      Ere revolving any complicated project, however, adapted to this end, I first simply suggested to Bartleby the propriety of his permanent departure. In a calm and serious tone, I commended the idea to his careful and mature consideration. But having taken three days to meditate upon it, he apprised me that his original determination remained the same; in short, that he still preferred to abide with me.

      What shall I do? I now said to myself, buttoning up my coat to the last button. What shall I do? what ought I to do? what does conscience say I should do with this man, or rather ghost. Rid myself of him, I must; go, he shall. But how? You will not thrust him, the poor, pale, passive mortal,--you will not thrust such a helpless creature out of your door? you will not dishonor yourself by such cruelty? No, I will not, I cannot do that. Rather would I let him live and die here, and then mason up his remains in the wall. What then will you do? For all your coaxing, he will not budge. Bribes he leaves under your own paperweight on your table; in short, it is quite plain that he prefers to cling to you.

      Then something severe, something unusual must be done. What! surely you will not have him collared by a constable, and commit his innocent pallor to the common jail? And upon what ground could you procure such a thing to be done?--a vagrant, is he? What! he a vagrant, a wanderer, who refuses to budge? It is because he will not be a vagrant, then, that you seek to count him as a vagrant. That is too absurd. No visible means of support: there I have him. Wrong again: for indubitably he does support himself, and that is the only unanswerable proof that any man can show of his possessing the means so to do. No more then. Since he will not quit me, I must quit him. I will change my offices; I will move elsewhere; and give him fair notice, that if I find him on my new premises I will then proceed against him as a common trespasser.

      Acting accordingly, next day I thus addressed him: "I find these chambers too far from the City Hall; the air is unwholesome. In a word, I propose to remove my offices next week, and shall no longer require your services. I tell you this now, in order that you may seek another place."

      He made no reply, and nothing more was said.

      On the appointed day I engaged carts and men, proceeded to my chambers, and having but little furniture, every thing was removed in a few hours. Throughout, the scrivener remained standing behind the screen, which I directed to be removed the last thing. It was withdrawn; and being folded up like a huge folio, left him the motionless occupant of a naked room. I stood in the entry watching him a moment, while something from within me upbraided me.

      I re-entered, with my hand in my pocket--and--and my heart in my mouth.

      "Good-bye, Bartleby; I am going--good-bye, and God some way bless you; and take that," slipping something in his hand. But it dropped to the floor, and then,--strange to say--I tore myself from him whom I had so longed to be rid of.

      Established in my new quarters, for a day or two I kept the door locked, and started at every footfall in the passages. When I returned to my rooms after any little absence, I would pause at the threshold for an instant, and attentively listen, ere applying my key. But these fears were needless. Bartleby never came nigh me.

      I thought all was going well, when a perturbed looking stranger visited me, inquiring whether I was the person who had recently occupied rooms at No.--Wall-street.

      Full of forebodings, I replied that I was.

      "Then, sir," said the stranger, who proved a lawyer, "you are responsible for the man you left there. He refuses to do any copying; he refuses to do any thing; he says he prefers not to; and he refuses to quit the premises."

      "I am very sorry, sir," said I, with assumed tranquillity, but an inward tremor, "but, really, the man you allude to is nothing to me --he is no relation or apprentice of mine, that you should hold me responsible for him."

      "In mercy's name, who is he?"

      "I certainly cannot inform you. I know nothing about him. Formerly I employed him as a copyist; but he has done nothing for me now for some time past."

      "I shall settle him then,--good morning, sir."

      Several days passed, and I heard nothing more; and though I often felt a charitable prompting to call at the place and see poor Bartleby, yet a certain squeamishness of I know not what withheld me.

      All is over with him, by this time, thought I at last, when through another week no further intelligence reached me. But coming to my room the day after, I found several persons waiting at my door in a high state of nervous excitement.

      "That's the man--here he comes," cried the foremost one, whom recognized as the lawyer who had previously called upon me alone.

      "You must take him away, sir, at once," cried a portly person among them, advancing upon me, and whom I knew to be the landlord of No.--Wall-street. "These gentlemen, my tenants, cannot stand it any longer; Mr. B--" pointing to the lawyer, "has turned him out of his room, and he now persists in haunting the buildinggenerally, sitting upon the banisters of the stairs by day, and sleeping in the entry by night. Every body is concerned; clients are leaving the offices; some fears are entertained of a mob; something you must do, and that without delay."

      Aghast at this torment, I fell back before it, and would fain have locked myselfin my new quarters. In vain I persisted that Bartleby was nothing to me--no more than to any one else. In vain:--I was the last person known to have any thing to do with him, and they held me to the terrible account. Fearful then of being exposed in the papers (as one person present obscurely threatened) I considered the matter, and at length said, that if the lawyer would give me a confidential interview with the scrivener, in his (the lawyer's) own room, I would that afternoon strive my best to rid them of the nuisance they complained of.

      Going up stairs to my old haunt, there was Bartleby silently sitting upon the banister at the landing.

      "What are you doing here, Bartleby?" said I.

      "Sitting upon the banister," he mildly replied.

      I motioned him into the lawyer's room, who then left us.

      "Bartleby," said I, "are you aware that you are the cause of great tribulation to me, by persisting in occupying the entry after being dismissed from the office?"

      No answer.

      "Now one of two things must take place. Either you must do something or something must be done to you. Now what sort of business would you like to engage in? Would you like to re-engage in copying for some one?"

      "No; I would prefer not to make any change."

      "Would you like a clerkship in a dry-goods store?"

      "There is too much confinement about that. No, I would not like a clerkship; but I am not particular."

      "Too much confinement," I cried, "why you keep yourself confined all the time!"

      "I would prefer not to take a clerkship," he rejoined, as if to settle that little item at once.

      "How would a bar-tender's business suit you? There is no trying of the eyesight in that."

      "I would not like it at all; though, as I said before, I am not particular."

      His unwonted wordiness inspirited me. I returned to the charge.

      "Well then, would you like to travel through the country collecting bills for the merchants? That would improve your health."

      "No, I would prefer to be doing something else."

      "How then would going as a companion to Europe, to entertain some young gentleman with your conversation,--how would that suit you?"

      "Not at all. It does not strike me that there is any thing definite about that. I like to be stationary. But I am not particular.

      "Stationary you shall be then," I cried, now losing all patience, and for the first time in all my exasperating connection with him fairly flying into a passion. "If you do not go away from these premises before night, I shall feel bound--indeed I am bound--to-- to--to quit the premises myself!" I rather absurdly concluded, knowing not with what possible threat to try to frighten his immobility into compliance. Despairing of all further efforts, I was precipitately leaving him, when a final thought occurred to me--one which had not been wholly unindulged before.

      "Bartleby," said I, in the kindest tone I could assume under such exciting circumstances, "will you go home with me now--not to my office, but my dwelling--and remain there till we can conclude upon some convenient arrangement for you at our leisure? Come, let us start now, right away."

      "No: at present I would prefer not to make any change at all."

      I answered nothing; but effectualy dodging every one by the suddenness and rapidity of my flight, rushed from the building, ran up Wall-street towards Broadway, and jumping into the first omnibus was soon removed from pursuit. As soon as tranquility returned I distinctly perceived that I had now done all that I possibly could, both in respect to the demands of the landlord and his tenants, and with regard to my own desire and sense of duty, to benefit Bartleby, and shield him from rude persecution. I now strove to be entirely care-free and quiescent; and my conscience justified me in the attempt; though indeed it was not so successful as I could have wished. So fearful was I of being again hunted out by the incensed landlord and his exasperated tenants, that, surrendering my business to Nippers, for a few days I drove about the upper part of the town and through the suburbs, in my rockaway; crossed over to Jersey City and Hoboken, and paid fugitive visits to Manhattanville and Astoria. In fact I almost lived in my rockaway for the time.

      When again I entered my office, lo, a note from the landlord lay upon desk. opened it with trembling hands. informed me that writer had sent to police, and Bartleby removed the Tombs as a vagrant. Moreover, since I knew more about him than any one else, he wished me to appear at that place, and make a suitable statement of the facts. These tidings had a conflicting effect upon me. At first I was indignant; but at last almost approved. The landlord's energetic, summary disposition, had led him to adopt a procedure which I do not think I would have decided upon myself; and yet as a last resort, under such peculiar circumstances, it seemed the only plan.

      As I afterwards learned, the poor scrivener, when told that he must be conducted to the Tombs, offered not the slightest obstacle, but in his pale unmoving way, silently acquiesced.

      Some of the compassionate and curious bystanders joined the party; and headed by one of the constables arm in arm with Bartleby, the silent procession filed its way through all the noise, and heat, and joy of the roaring thoroughfares at noon.

      The same day I received the note I went to the Tombs, or to speak more properly, the Halls of Justice. Seeking the right officer, I stated the purpose of my call, and was informed that the individual I described was indeed within. I then assured the functionary that Bartleby was a perfectly honest man, and greatly to be compassionated, however unaccountably eccentric. I narrated all I knew,and closed by suggesting the idea of letting him remain in as indulgent confinement as possible till something less harsh might be done--though indeed I hardly knew what. At all events, if nothing else could be decided upon, the alms-house must receive him. I then begged to have an interview.

      Being under no disgraceful charge, and quite serene and harmless in all his ways, they had permitted him freely to wander about the prison, and especially in the inclosed grass-platted yards thereof. And so I found him there, standing all alone in the quietest of the yards, his face towards a high wall, while all around, from the narrow slits of the jail windows, I thought I saw peering out upon him the eyes of murderers and thieves.

      "Bartleby!"

      "I know you," he said, without looking round,--"and I want nothing to say to you."

      "It was not I that brought you here, Bartleby," said I, keenly pained at his implied suspicion. "And to you, this should not be so vile a place. Nothing reproachful attaches to you by being here. And see, it is not so sad a place as one might think. Look, there is the sky, and here is the grass."

      "I know where I am," he replied, but would say nothing more, and so I left him.

      As I entered the corridor again, a broad meat-like man in an apron, accosted me, and jerking his thumb over his shoulder said--"Is that your friend?"

      "Yes."

      "Does he want to starve? If he does, let him live on the prison fare, that's all.

      "Who are you?" asked I, not knowing what to make of such an unofficially speaking person in such a place.

      "I am the grub-man. Such gentlemen as have friends here, hire me to provide them with something good to eat."

      "Is this so?" said I, turning to the turnkey.

      He said it was.

      "Well then," said I, slipping some silver into the grub-man's hands (for so they called him). "I want you to give particular attention to my friend there; let him have the best dinner you can get. And you must be as polite to him as possible."

      "Introduce me, will you?" said the grub-man, looking at me with an expression which seemed to say he was all impatience for an opportunity to give a specimen of his breeding.

      Thinking it would prove of benefit to the scrivener, I acquiesced; and asking the grub-man his name, went up with him to Bartleby.

      "Bartleby, this is a friend; you will find him very useful to you."

      "Your sarvant, sir, your sarvant," said the grub-man, making a low salutation behind his apron. "Hope you find it pleasant here, sir;--spacious grounds--cool apartments, sir--hope you'll stay with us some time--try to make it agreeable. What will you have for dinner today?"

      "I prefer not to dine to-day," said Bartleby, turning away. "It would disagree with me; I am unused to dinners." So saying he slowly moved to the other side of the inclosure, and took up a position fronting the dead-wall.

      "How's this?" said the grub-man, addressing me with a stare of astonishment. "He's odd, aint he?"

      "I think he is a little deranged," said I, sadly.

      "Deranged? deranged is it? Well now, upon my word, I thought that friend of yourn was a gentleman forger; they are always pale and genteel-like, them forgers. I can't help pity 'em--can't help it, sir. Did you know Monroe Edwards?" he added touchingly, and paused. Then, laying his hand pityingly on my shoulder, sighed, "he died of consumption at Sing-Sing. so you weren't acquainted with Monroe?"

      "No, I was never socially acquainted with any forgers. But I cannot stop longer. Look to my friend yonder. You will not lose by it. I will see you again."

      Some few days after this, I again obtained admission to the Tombs, and went through the corridors in quest of Bartleby; but without finding him.

      "I saw him coming from his cell not long ago," said a turnkey, "may be he's gone to loiter in the yards."

      So I went in that direction.

      "Are you looking for the silent man?" said another turnkey passing me. "Yonder he lies--sleeping in the yard there. 'Tis not twenty minutes since I saw him lie down."

      The yard was entirely quiet. It was not accessible to the common prisoners. The surrounding walls, of amazing thickness, kept off all sound behind them. The Egyptian character of the masonry weighed upon me with its gloom. But a soft imprisoned turf grew under foot. The heart of the eternal pyramids, it seemed, wherein, by some strange magic, through the clefts, grass-seed, dropped by birds, had sprung.

      Strangely huddled at the base of the wall, his knees drawn up, and lying on his side, his head touching the cold stones, I saw the wasted Bartleby. But nothing stirred. I paused; then went close up to him; stooped over, and saw that his dim eyes were open; otherwise he seemed profoundly sleeping. Something prompted me to touch him. I felt his hand, when a tingling shiver ran up my arm and down my spine to my feet.

      The round face of the grub-man peered upon me now. "His dinner is ready. Won't he dine to-day, either? Or does he live without dining?"

      "Lives without dining," said I, and closed the eyes.

      "Eh!--He's asleep, aint he?"

      "With kings and counsellors," murmured I.

      * * * * * * * *

      There would seem little need for proceeding further in this history. Imagination will readily supply the meagre recital of poor Bartleby's interment. But ere parting with the reader, let me say, that if this little narrative has sufficiently interested him, to awaken curiosity as to who Bartleby was, and what manner of life he led prior to the present narrator's making his acquaintance, I can only reply, that in such curiosity I fully share, but am wholly unable to gratify it. Yet here I hardly know whether I should divulge one little item of rumor, which came to my ear a few months after the scrivener's decease. Upon what basis it rested, I could never ascertain; and hence how true it is I cannot now tell. But inasmuch as this vague report has not been without a certain strange suggestive interest to me, however said, it may prove the same with some others; and so I will briefly mention it. The report was this: that Bartleby had been a subordinate clerk in the Dead Letter Office at Washington, from which he had been suddenly removed by a change in the administration. When I think over this rumor, I cannot adequately express the emotions which seize me. Dead letters! does it not sound like dead men? Conceive a man by nature and misfortune prone to a pallid hopelessness, can any business seem more fitted to heighten it than that of continually handling these dead letters and assorting them for the flames? For by the cart-load they are annually burned. Sometimes from out the folded paper the pale clerk takes a ring:--the bank-note sent in swiftest charity:--he whom it would relieve, nor eats nor hungers any more; pardon for those who died despairing; hope for those who died unhoping; good tidings for those who died stifled by unrelieved calamities. On errands of life, these letters speed to death.

      Ah Bartleby! Ah humanity!

      Text Without Notes
      Melville Web Links

      ================================================ FILE: test/test-pages/cnet/expected-metadata.json ================================================ { "title": "Zuckerberg offers peek at Facebook's acquisition strategies", "byline": "Steven Musil", "dir": null, "lang": "en", "excerpt": "Facebook CEO says be a friend and have a shared vision, but scare them when you have to and move fast.", "siteName": "CNET", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/cnet/expected.html ================================================

      Facebook CEO Mark Zuckerberg, the man with the acquisition plan.

      Photo by James Martin/CNET

      Anyone who has ever been involved in closing a billion-dollar acquisition deal will tell you that you don't go in without a clear, well thought out plan.

      Facebook CEO Mark Zuckerberg knows a thing or two about how to seal the deal on blockbuster buys. After all, he's the man behind his company's $19 billion acquisition of WhatsApp, he personally brokered its $1 billion buyout of Instagram and closed the $3 billion deal to buy Oculus VR.

      Zuckerberg offered a primer on the strategies he and his company employ when they see an attractive target during testimony Tuesday in a lawsuit with ZeniMax Media, which accuses Oculus and Facebook of "misappropriating" trade secrets and copyright infringement. At the heart of the lawsuit is technology that helped create liftoff for virtual reality, one of the hottest gadget trends today.

      A key Facebook approach is building a long-term relationship with your target, Zuckerberg said at the trial. These deals don't just pop up over night, he said according to a transcript reviewed by Business Insider. They take time to cultivate.

      I've been building relationships, at least in Instagram and the WhatsApp cases, for years with the founders and the people that are involved in these companies, which made [it] so that when it became time or when we thought it was the right time to move, we felt like we had a good amount of context and had good relationships so that we could move quickly, which was competitively important and why a lot of these acquisitions, I think, came to us instead of our competitors and ended up being very good acquisitions over time that a lot of competitors wished they had gotten instead.

      He also stressed the need assure your target that you have a shared vision about how you will collaborate after the deal is put to bed. Zuckerberg said this was reason Facebook was able to acquire Oculus for less than its original $4 billion asking price.

      If this [deal] is going to happen, it's not going to be because we offer a lot of money, although we're going to have to offer a fair price for the company that is more than what they felt like they could do on their own. But they also need to feel like this was actually going to help their mission.

      When that doesn't work, Zuckerberg said scare tactics is an effective, if undesirable, way of persuading small startups that they face a better chance of survival if they have Facebook to guide their way rather than going it alone.

      That's less my thing, but I think if you are trying to help convince people that they want to join you, helping them understand all the pain that they would have to go through to build it out independently is a valuable tactic.

      It also pays to be weary of competing suitors for your startup, Zuckerberg said, and be willing to move fast to stave off rivals and get the deal done.

      Often, if a company knows we're offering something, they will offer more. So being able to move quickly not only increases our chance of being able to get a deal done if we want to, but it makes it so we don't have end up having to pay a lot more because the process drags out.

      It wasn't clear why these strategies didn't work on Snapchat CEO Evan Spiegel, who famously rebuffed a $3 billion takeover offer from Facebook in 2013.

      Tech Enabled: CNET chronicles tech's role in providing new kinds of accessibility. Check it out here.

      Technically Literate: Original works of short fiction with unique perspectives on tech, exclusively on CNET. Here.

      ================================================ FILE: test/test-pages/cnet/source.html ================================================ Zuckerberg offers peek at Facebook's acquisition strategies - CNET

      Zuckerberg offers peek at Facebook's acquisition strategies

      Facebook CEO says be a friend and have a shared vision, but scare them when you have to and move fast.

      Tech Industry
      Up Next Governments suck at social media, but you deserve some blame

      Facebook CEO Mark Zuckerberg, the man with the acquisition plan.

      Photo by James Martin/CNET

      Anyone who has ever been involved in closing a billion-dollar acquisition deal will tell you that you don't go in without a clear, well thought out plan.

      Facebook CEO Mark Zuckerberg knows a thing or two about how to seal the deal on blockbuster buys. After all, he's the man behind his company's $19 billion acquisition of WhatsApp, he personally brokered its $1 billion buyout of Instagram and closed the $3 billion deal to buy Oculus VR.

      Zuckerberg offered a primer on the strategies he and his company employ when they see an attractive target during testimony Tuesday in a lawsuit with ZeniMax Media, which accuses Oculus and Facebook of "misappropriating" trade secrets and copyright infringement. At the heart of the lawsuit is technology that helped create liftoff for virtual reality, one of the hottest gadget trends today.

      A key Facebook approach is building a long-term relationship with your target, Zuckerberg said at the trial. These deals don't just pop up over night, he said according to a transcript reviewed by Business Insider. They take time to cultivate.

      I've been building relationships, at least in Instagram and the WhatsApp cases, for years with the founders and the people that are involved in these companies, which made [it] so that when it became time or when we thought it was the right time to move, we felt like we had a good amount of context and had good relationships so that we could move quickly, which was competitively important and why a lot of these acquisitions, I think, came to us instead of our competitors and ended up being very good acquisitions over time that a lot of competitors wished they had gotten instead.

      He also stressed the need assure your target that you have a shared vision about how you will collaborate after the deal is put to bed. Zuckerberg said this was reason Facebook was able to acquire Oculus for less than its original $4 billion asking price.

      If this [deal] is going to happen, it's not going to be because we offer a lot of money, although we're going to have to offer a fair price for the company that is more than what they felt like they could do on their own. But they also need to feel like this was actually going to help their mission.

      When that doesn't work, Zuckerberg said scare tactics is an effective, if undesirable, way of persuading small startups that they face a better chance of survival if they have Facebook to guide their way rather than going it alone.

      That's less my thing, but I think if you are trying to help convince people that they want to join you, helping them understand all the pain that they would have to go through to build it out independently is a valuable tactic.

      It also pays to be weary of competing suitors for your startup, Zuckerberg said, and be willing to move fast to stave off rivals and get the deal done.

      Often, if a company knows we're offering something, they will offer more. So being able to move quickly not only increases our chance of being able to get a deal done if we want to, but it makes it so we don't have end up having to pay a lot more because the process drags out.

      It wasn't clear why these strategies didn't work on Snapchat CEO Evan Spiegel, who famously rebuffed a $3 billion takeover offer from Facebook in 2013.

      Tech Enabled: CNET chronicles tech's role in providing new kinds of accessibility. Check it out here.

      Technically Literate: Original works of short fiction with unique perspectives on tech, exclusively on CNET. Here.

      Up Next: Governments suck at social media, but you deserve some blame
      Close
      Battle of the CES robot baristas
      Drag
      Autoplay: ON Autoplay: OFF
      x
      player version 1.7.2
      playback state 1
      duration 15
      current time 7.00
      buffer length 0.00
      average dropped (fps) 0
      • playlist
      • 0:07 / 0:15
      • captions
      • fullscreen
      Special feature

      How to build worlds for fun and profit

      by
      Exclusive interview

      Does the Mac still matter?

      by

      CNET Magazine

      Read about tech’s impact on the worldwide refugee crisis, discover what conservationists, scientists and tech companies are doing to save the rhino, learn how Tetris conquered the world, and more in the Winter issue of CNET Magazine.

      Grab your copy
      Close
      BECOME A CNET MEMBER FOR FREE!

      Share your ideas with the community, receive exclusive offers for sweeps and events, and get the latest news, reviews and how-tos in your inbox.

      No, thanks.
      ================================================ FILE: test/test-pages/cnet-svg-classes/expected-metadata.json ================================================ { "title": "Twitter Lite se estrena en México, Venezuela y otros nueve países", "byline": "César Salza", "dir": null, "lang": "es", "excerpt": "Twitter Lite llega a 11 países de América Latina, para ayudar a los usuarios con mala señal de sus redes móviles.", "siteName": "CNET en Español", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/cnet-svg-classes/expected.html ================================================
      twitter-lite

      Twitter Lite estará disponible en Google Play Store en 11 países de América Latina.

       Twitter

      Twitter ha dado a conocer que Twitter Lite llegará a un total de 24 nuevos países a partir de hoy, 11 de ellos de América Latina.

      Según explicó en un comunicado Twitter Lite ahora estará disponible en Bolivia, Brasil, Chile, Colombia, Costa Rica, Ecuador, México, Panamá, Perú, El Salvador y Venezuela.

      Twitter Lite es la versión ligera de la aplicación de la red social para Android, disponible en la Google Play Store. Con este app los usuarios que experimentan fallos de red o que viven en países con redes con poca velocidad de conexión como Venezuela podrán descargar los tuits de forma más rápida.

      Entre sus novedades, Twitter Lite permite la carga rápida de tuits en redes 2G y 3G, y ofrece ayuda offline en caso de que pierdas tu conexión; a eso debemos sumar que minimiza el uso de datos y ofrece un modo de ahorro, en el que únicamente se descargan las fotos o videos de los tuits que quieres ver.

      Además, el app ocupa menos espacio en tu teléfono móvil, al reducir a 3MB su peso.

      Twitter dio a conocer Twitter Lite en abril en India, y desde entonces ha estado trabajando para llevarlo a más países. La empresa en los últimos meses también se ha involucrado de forma definitiva en la eliminación de los abusos en la red social, tomando medidas incluso en la verificación de cuentas.

      Reproduciendo: Mira esto: Google Assistant mejora, hay más cambios en Twitter y...

      8:09

      ================================================ FILE: test/test-pages/cnet-svg-classes/source.html ================================================ Twitter Lite se estrena en México, Venezuela y otros nueve países - CNET en Español

      CNET también está disponible en español.

      Ir a español

      Don't show this again

      guía de compras navideñas
      Aplicaciones móviles

      Twitter Lite se estrena en 11 países de América Latina

      La aplicación de Twitter que minimiza el uso de datos y permite la carga de tuits de forma más rápida llega a México y Venezuela.

      twitter-lite

      Twitter Lite estará disponible en Google Play Store en 11 países de América Latina.

       Twitter

      Twitter ha dado a conocer que Twitter Lite llegará a un total de 24 nuevos países a partir de hoy, 11 de ellos de América Latina.

      Según explicó en un comunicado Twitter Lite ahora estará disponible en Bolivia, Brasil, Chile, Colombia, Costa Rica, Ecuador, México, Panamá, Perú, El Salvador y Venezuela.

      Twitter Lite es la versión ligera de la aplicación de la red social para Android, disponible en la Google Play Store. Con este app los usuarios que experimentan fallos de red o que viven en países con redes con poca velocidad de conexión como Venezuela podrán descargar los tuits de forma más rápida.

      Artículos Relacionados

      Entre sus novedades, Twitter Lite permite la carga rápida de tuits en redes 2G y 3G, y ofrece ayuda offline en caso de que pierdas tu conexión; a eso debemos sumar que minimiza el uso de datos y ofrece un modo de ahorro, en el que únicamente se descargan las fotos o videos de los tuits que quieres ver.

      Además, el app ocupa menos espacio en tu teléfono móvil, al reducir a 3MB su peso.

      Twitter dio a conocer Twitter Lite en abril en India, y desde entonces ha estado trabajando para llevarlo a más países. La empresa en los últimos meses también se ha involucrado de forma definitiva en la eliminación de los abusos en la red social, tomando medidas incluso en la verificación de cuentas.

      Reproduciendo: Mira esto: Google Assistant mejora, hay más cambios en Twitter y...
      8:09
      Siguiente artículo: Si quieres un iPhone X, ya no tendrás que esperar tanto
      ================================================ FILE: test/test-pages/cnn/expected-metadata.json ================================================ { "title": "The 'birth lottery' and economic mobility", "byline": "Ahiza Garcia", "dir": null, "excerpt": "A recently-released report on poverty and inequality found that the U.S. ranks the lowest among countries with welfare states.", "siteName": "CNNMoney", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/cnn/expected.html ================================================

      The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.

      But a new report released on Monday by Stanford University's Center on Poverty and Inequality calls that into question.

      The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs.

      Powered by SmartAsset.com

      Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries.

      As the report states: "[T]he birth lottery matters more in the U.S. than in most well-off countries."

      But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work.

      Related: Rich are paying more in taxes but not as much as they used to

      The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries.

      It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country.

      The overall rankings of the countries were as follows:
      1. Finland
      2. Norway
      3. Australia
      4. Canada
      5. Germany
      6. France
      7. United Kingdom
      8. Italy
      9. Spain
      10. United States

      The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty.

      Related: Chicago is America's most segregated city

      The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries.

      CNNMoney (New York) First published February 1, 2016: 1:28 AM ET

      ================================================ FILE: test/test-pages/cnn/source.html ================================================ The 'birth lottery' and economic mobility - Feb. 1, 2016
      American Opportunity

      The 'birth lottery' and economic mobility

      The priest saving LA's gang members
      Your video will play in 00:25

      The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.

      But a new report released on Monday by Stanford University's Center on Poverty and Inequality calls that into question.

      The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs.

      Powered by SmartAsset.com

      Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries.

      As the report states: "[T]he birth lottery matters more in the U.S. than in most well-off countries."

      But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work.

      Related: Rich are paying more in taxes but not as much as they used to

      The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries.

      It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country.

      The overall rankings of the countries were as follows:
      1. Finland
      2. Norway
      3. Australia
      4. Canada
      5. Germany
      6. France
      7. United Kingdom
      8. Italy
      9. Spain
      10. United States

      The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty.

      Related: Chicago is America's most segregated city

      The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries.

      CNNMoney (New York) First published February 1, 2016: 1:28 AM ET

      Social Surge - What's Trending

      Mortgage & Savings

      Terms & Conditions apply

      NMLS #1136

      Search for Jobs

      LendingTree

      Newsletter

      CNNMoney Sponsors

      Partner Offers

      NextAdvisor

      ================================================ FILE: test/test-pages/comment-inside-script-parsing/expected-metadata.json ================================================ { "title": "Test script parsing", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/comment-inside-script-parsing/expected.html ================================================

      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

      Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      ================================================ FILE: test/test-pages/comment-inside-script-parsing/source.html ================================================ Test script parsing

      Lorem

      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

      Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      Foo

      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      ================================================ FILE: test/test-pages/daringfireball-1/expected-metadata.json ================================================ { "title": "Daring Fireball: Colophon", "byline": null, "dir": null, "lang": "en", "excerpt": "Daring Fireball is written and produced by John Gruber.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/daringfireball-1/expected.html ================================================

      About This Site

      Daring Fireball is written and produced by John Gruber.

      Photograph of the author.
      Portrait by George Del Barrio

      Mac Apps

      iPhone Apps

      Server Software

      The Daring Fireball website is hosted by Joyent.

      Articles and links are published through Movable Type. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s MT-Regex and MT-IfEmpty, and Nat Irons’s Amputator.

      Stats are tracked using Mint. Additional web nerdery, including the membership system, is fueled by Perl, PHP, and MySQL.

      Web Standards

      Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either HTML 5 or XHTML 4.01 Transitional, its layout is constructed using valid CSS, and its syndicated feed is valid Atom.

      If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.

      ================================================ FILE: test/test-pages/daringfireball-1/source.html ================================================ Daring Fireball: Colophon

      About This Site

      Daring Fireball is written and produced by John Gruber.

      Photograph of the author.
      Portrait by George Del Barrio

      Mac Apps

      iPhone Apps

      Server Software

      The Daring Fireball website is hosted by Joyent.

      Articles and links are published through Movable Type. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s MT-Regex and MT-IfEmpty, and Nat Irons’s Amputator.

      Stats are tracked using Mint. Additional web nerdery, including the membership system, is fueled by Perl, PHP, and MySQL.

      Web Standards

      Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either HTML 5 or XHTML 4.01 Transitional, its layout is constructed using valid CSS, and its syndicated feed is valid Atom.

      If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.

      Ads via The Deck Ads via The Deck

      ================================================ FILE: test/test-pages/data-url-image/expected-metadata.json ================================================ { "title": "Document", "byline": null, "dir": null, "lang": "en", "excerpt": "Lorem ipsum dolor sit amet consectetur adipisicing elit. Natus eaque totam provident obcaecati nisi praesentium iusto velit fuga debitis quidem ut repellat corrupti, eligendi inventore quibusdam perspiciatis delectus omnis pariatur excepturi quasi fugit? A adipisci natus nostrum, qui aperiam, at culpa corrupti autem enim earum vitae. Nostrum et officiis facere ex recusandae tenetur, delectus odit provident soluta id perferendis ducimus quibusdam corporis rerum voluptatem architecto sequi beatae quod mollitia voluptatibus earum tempora inventore ut. Deserunt reprehenderit recusandae nostrum, eaque fuga cum, repellat, perspiciatis ducimus in non consequatur ratione. Sint rerum necessitatibus deleniti odio earum voluptatum eos modi ab dolor minus.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/data-url-image/expected.html ================================================
      Illustration for article titled The Spectacular Story Of Metroid

      Lorem ipsum dolor sit amet consectetur adipisicing elit. Natus eaque totam provident obcaecati nisi praesentium iusto velit fuga debitis quidem ut repellat corrupti, eligendi inventore quibusdam perspiciatis delectus omnis pariatur excepturi quasi fugit? A adipisci natus nostrum, qui aperiam, at culpa corrupti autem enim earum vitae. Nostrum et officiis facere ex recusandae tenetur, delectus odit provident soluta id perferendis ducimus quibusdam corporis rerum voluptatem architecto sequi beatae quod mollitia voluptatibus earum tempora inventore ut. Deserunt reprehenderit recusandae nostrum, eaque fuga cum, repellat, perspiciatis ducimus in non consequatur ratione. Sint rerum necessitatibus deleniti odio earum voluptatum eos modi ab dolor minus.

      Illustration for article titled The Spectacular Story Of Metroid

      Quae veniam recusandae vel cupiditate doloribus pariatur, dolorum saepe hic quos mollitia harum nihil molestias magni modi maiores? Ea accusamus velit vel doloremque pariatur voluptate? Consequuntur dolorum fugit facere architecto eveniet beatae eligendi rerum nisi maxime, voluptatum a repellendus perferendis dolores eius repudiandae accusamus animi eum esse adipisci voluptatibus incidunt necessitatibus! Adipisci id, quos libero dolor odit dignissimos et, cum explicabo est facere aliquid mollitia! Iure nihil nobis beatae quibusdam, dolor saepe reiciendis, exercitationem hic accusamus quasi commodi modi molestiae ipsum, maxime non nesciunt deserunt itaque quo cupiditate rerum ipsa ad minus vel quod. Earum dolorum velit aut minus hic?

      Quidem nam tempore ex harum repellat modi expedita quia rerum est ut ullam, quisquam repudiandae at dolorum. Sint atque fugit facere id blanditiis dolor omnis rem repudiandae, laboriosam unde delectus illo voluptate dolorem consectetur rerum expedita quas debitis deleniti iure amet. At sequi distinctio facere eaque, esse molestiae tenetur, unde sapiente fugit minus eos ex aperiam atque? Animi soluta, dolorem sequi doloremque pariatur in. Impedit, voluptates dolorem natus perspiciatis pariatur eligendi labore rerum facilis est error. Illum eveniet nostrum atque nobis molestiae expedita perferendis, esse ullam corrupti sit explicabo beatae. Sapiente aliquid nesciunt reiciendis incidunt cumque et officiis minus aliquam?

      Veritatis sit nostrum officiis alias ea quam nulla veniam eos, quidem nisi, magni molestias ut odio. Ea, mollitia nesciunt. Maiores iste quos quas sapiente sit at esse quidem? Laboriosam fugiat nobis voluptatum reiciendis aliquam excepturi ipsum? Perspiciatis expedita sit quod, optio assumenda veritatis culpa, neque explicabo distinctio facere incidunt magnam accusamus, quidem animi delectus doloremque vel molestiae a quae dolorem. Sit, at? Numquam possimus animi esse incidunt quos quibusdam. Ab sit eligendi laborum beatae maxime suscipit, obcaecati enim cum nam autem, dicta nobis corrupti ut aut nostrum ex excepturi perferendis, soluta veniam. Excepturi beatae suscipit enim dignissimos! Quam, esse praesentium!

      ================================================ FILE: test/test-pages/data-url-image/source.html ================================================ Document Illustration for article titled The Spectacular Story Of Metroid

      Lorem ipsum dolor sit amet consectetur adipisicing elit. Natus eaque totam provident obcaecati nisi praesentium iusto velit fuga debitis quidem ut repellat corrupti, eligendi inventore quibusdam perspiciatis delectus omnis pariatur excepturi quasi fugit? A adipisci natus nostrum, qui aperiam, at culpa corrupti autem enim earum vitae. Nostrum et officiis facere ex recusandae tenetur, delectus odit provident soluta id perferendis ducimus quibusdam corporis rerum voluptatem architecto sequi beatae quod mollitia voluptatibus earum tempora inventore ut. Deserunt reprehenderit recusandae nostrum, eaque fuga cum, repellat, perspiciatis ducimus in non consequatur ratione. Sint rerum necessitatibus deleniti odio earum voluptatum eos modi ab dolor minus.

      Illustration for article titled The Spectacular Story Of Metroid

      Quae veniam recusandae vel cupiditate doloribus pariatur, dolorum saepe hic quos mollitia harum nihil molestias magni modi maiores? Ea accusamus velit vel doloremque pariatur voluptate? Consequuntur dolorum fugit facere architecto eveniet beatae eligendi rerum nisi maxime, voluptatum a repellendus perferendis dolores eius repudiandae accusamus animi eum esse adipisci voluptatibus incidunt necessitatibus! Adipisci id, quos libero dolor odit dignissimos et, cum explicabo est facere aliquid mollitia! Iure nihil nobis beatae quibusdam, dolor saepe reiciendis, exercitationem hic accusamus quasi commodi modi molestiae ipsum, maxime non nesciunt deserunt itaque quo cupiditate rerum ipsa ad minus vel quod. Earum dolorum velit aut minus hic?

      Quidem nam tempore ex harum repellat modi expedita quia rerum est ut ullam, quisquam repudiandae at dolorum. Sint atque fugit facere id blanditiis dolor omnis rem repudiandae, laboriosam unde delectus illo voluptate dolorem consectetur rerum expedita quas debitis deleniti iure amet. At sequi distinctio facere eaque, esse molestiae tenetur, unde sapiente fugit minus eos ex aperiam atque? Animi soluta, dolorem sequi doloremque pariatur in. Impedit, voluptates dolorem natus perspiciatis pariatur eligendi labore rerum facilis est error. Illum eveniet nostrum atque nobis molestiae expedita perferendis, esse ullam corrupti sit explicabo beatae. Sapiente aliquid nesciunt reiciendis incidunt cumque et officiis minus aliquam?

      Veritatis sit nostrum officiis alias ea quam nulla veniam eos, quidem nisi, magni molestias ut odio. Ea, mollitia nesciunt. Maiores iste quos quas sapiente sit at esse quidem? Laboriosam fugiat nobis voluptatum reiciendis aliquam excepturi ipsum? Perspiciatis expedita sit quod, optio assumenda veritatis culpa, neque explicabo distinctio facere incidunt magnam accusamus, quidem animi delectus doloremque vel molestiae a quae dolorem. Sit, at? Numquam possimus animi esse incidunt quos quibusdam. Ab sit eligendi laborum beatae maxime suscipit, obcaecati enim cum nam autem, dicta nobis corrupti ut aut nostrum ex excepturi perferendis, soluta veniam. Excepturi beatae suscipit enim dignissimos! Quam, esse praesentium!

      ================================================ FILE: test/test-pages/dev418/expected-metadata.json ================================================ { "title": "Readability Test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/dev418/expected.html ================================================

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Single <img>

      An image

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Single <figure>

      An image
      Caption of the figure

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      <ul> List of <img>

      • An image
      • An image
      • An image

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      <ul> List of <figure>

      • An image
        Caption of the figure
      • An image
        Caption of the figure
      • An image
        Caption of the figure

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      ================================================ FILE: test/test-pages/dev418/source.html ================================================ Readability Test

      Readability Test

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Single <img>

      An image

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      Single <figure>

      An image
      Caption of the figure

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      <ul> List of <img>

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      <ul> List of <figure>

      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

      ================================================ FILE: test/test-pages/dropbox-blog/expected-metadata.json ================================================ { "title": "How we designed Dropbox’s ATF - an async task framework", "byline": "Arun Sai Krishnan", "dir": null, "lang": "en", "excerpt": "I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/dropbox-blog/expected.html ================================================

      I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.

      This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.

      ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services. 

      As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.

      Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.

      Introduction

      Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.

      Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling. 

      Glossary

      Some basic terms repeatedly used in this post, defined as used in the context of this discussion.

      Lambda: A callback implementing business logic.

      Task: Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.

      Collection: A labeled subset of tasks belonging to a lambda. If send email is implemented as a lambda, then password reset email and marketing email would be collections.

       Priority: Labels defining priority of execution of tasks within a lambda. 

      Features

      Task scheduling
      Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.

      Priority based execution
      Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.

      Task gating
      ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.

      Track task status
      Clients can query the status of a scheduled task.

      System guarantees

      At-least once task execution
       The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.

      No concurrent task execution
       The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.

      Isolation
       Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.

      Delivery latency
       95% of tasks begin execution within five seconds from their scheduled execution time.

      High availability for task scheduling
       The ATF service is 99.9% available to accept task scheduling requests from any client.

      Lambda requirements

      Following are some restrictions we place on the callback logic (lambda):

      Idempotence
      A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.

      Resiliency
      Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.

      Terminal state handling
       ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries. 

      Architecture

      Async Task Framework (ATF) [Fig 1]
      Async Task Framework (ATF) [Fig 1]

      In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox uses gRPC for remote calls and our in-house Edgestore to store tasks.

      ATF consists of the following components: 

      • Frontend
      • Task Store
      • Store Consumer
      • Queue
      • Controller
      • Executor
      • Heartbeat and Status Controller (HSC)

      Frontend
      This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.

      Task Store
       ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be found in the Data Model section below.

      Store Consumer
       The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system. 

      Below is a simple walkthrough of the Store Consumer’s function:  

      repeat every second:
  1. poll tasks ready for execution from task store
  2. push tasks onto the right queues
  3. update task statuses

      The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the Lifecycle of a task section below.

      Queue
       ATF uses AWS Simple Queue Service (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <lambda, priority>  pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is #lambdas x #priorities.

      Controller
       Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues. 

      The Controller serves tasks from its process local queue as a response to NextWork RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to NextWork RPCs.

      Executor
       The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop: 

      while True:
  w = get_next_work()
  do_work(w)

      Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.

      Heartbeat and Status Controller (HSC)
      The HSC serves RPCs for claiming a task for execution (ClaimTask), setting task status after execution (SetResults) and heartbeats during task execution (Heartbeat). ClaimTask requests originate from the Controllers in response to NextWork requests. Heartbeat and SetResults requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.

      Data model

      ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (assoc), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.

      Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.

      Lifecycle of a task

      1. Client performs a Schedule RPC call to Frontend with task information, including execution time. 
      2. Frontend creates Edgestore entity and assoc for the task. 
      3. When it is time to process the task, Store Consumer pulls the task from Edgestore and pushes it to a related SQS queue. 
      4. Executor makes NextWork RPC call to Controller, which pulls tasks from the SQS queue, makes a ClaimTask RPC to the HSC and then returns the task to the Executor. 
      5. Executor invokes the callback for the task. While processing, Executor performs Heartbeat RPC calls to Heartbeat and Status Controller (HSC). Once processing is done, Executor performs TaskStatus RPC call to HSC. 
      6. Upon getting Heartbeat and TaskStatus RPC calls, HSC updates the Edgestore entity and assoc.

      Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the assoc. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.

      Following are the task entity and association states in ATF and their corresponding timestamp updates:

      Entity status

      Assoc status

      next trigger timestamp in Assoc

      Comment

      new

      new

      scheduled_timestamp of the task

      Pick up new tasks that are ready. 

      enqueued

      started

      enqueued_timestamp + enqueue_timeout

      Re-enqueue task if it has been in enqueued state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.

      claimed

      started

      claimed_timestamp + claim_timeout

      Re-enqueue if task is claimed but never transfered to processing. This can happen if Controller is down after claiming a task. Task status is changed to enqueued after re-enqueue.

      processing

      started

      heartbeat_timestamp + heartbeat_timeout`

      Re-enqueue if task hasn’t sent heartbeat for too long. This can happen if Executor is down. Task status is changed to enqueued after re-enqueue. 

      retriable failure

      started

      compute next_timestamp according to backoff logic

      Exponential backoff for tasks with retriable failure. 

      success

      completed

      N/A

      fatal_failure

      completed

      N/A

      The store consumer polls for tasks based on the following query:

      assoc_status= && next_timestamp<=time.now()

      Below is the state machine that defines task state transitions: 

      Task State Transitions [Fig 2]

      Achieving guarantees

      At-least-once task execution
       At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a Success or a FatalFailure state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a RetriableFailure state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.

      No concurrent task execution
       Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (Claimed) before starting execution. Once the task execution is complete, the task status is updated to one of Success, FatalFailure or RetriableFailure. A task can be claimed only if its existing task state is Enqueued (retried tasks go to the Enqueued state as well once they are re-pushed onto SQS).

      However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution. 

      To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.

      Isolation
       Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.

      Delivery latency
       ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.

      Ownership model

      ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.

      Extending ATF

      As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF. 

      Periodic task execution
       Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.

      Better support for task chaining
       Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.

      Dead letter queues for misbehaving tasks
       One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.

      One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.

      Conclusion

      We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.
       

      ================================================ FILE: test/test-pages/dropbox-blog/source.html ================================================ How we designed Dropbox ATF: an async task framework - Dropbox
      Dropbox.Tech

      // Press enter to search

      How we designed Dropbox ATF: an async task framework

      // By Arun Sai Krishnan • Nov 11, 2020

      I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.

      This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.

      ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services. 

      As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.

      Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.

      Introduction

      Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.

      Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling. 

      Glossary

      Some basic terms repeatedly used in this post, defined as used in the context of this discussion.

      Lambda: A callback implementing business logic.

      Task: Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.

      Collection: A labeled subset of tasks belonging to a lambda. If send email is implemented as a lambda, then password reset email and marketing email would be collections.

       Priority: Labels defining priority of execution of tasks within a lambda. 

      Features

      Task scheduling
      Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.

      Priority based execution
      Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.

      Task gating
      ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.

      Track task status
      Clients can query the status of a scheduled task.

      System guarantees

      At-least once task execution
       The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.

      No concurrent task execution
       The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.

      Isolation
       Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.

      Delivery latency
       95% of tasks begin execution within five seconds from their scheduled execution time.

      High availability for task scheduling
       The ATF service is 99.9% available to accept task scheduling requests from any client.

      Lambda requirements

      Following are some restrictions we place on the callback logic (lambda):

      Idempotence
      A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.

      Resiliency
      Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.

      Terminal state handling
       ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries. 

      Architecture

      Async Task Framework (ATF) [Fig 1]
      Async Task Framework (ATF) [Fig 1]

      In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox uses gRPC for remote calls and our in-house Edgestore to store tasks.

      ATF consists of the following components: 

      • Frontend
      • Task Store
      • Store Consumer
      • Queue
      • Controller
      • Executor
      • Heartbeat and Status Controller (HSC)

      Frontend
      This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.

      Task Store
       ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be found in the Data Model section below.

      Store Consumer
       The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system. 

      Below is a simple walkthrough of the Store Consumer’s function:  

      repeat every second:
  1. poll tasks ready for execution from task store
  2. push tasks onto the right queues
  3. update task statuses

      The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the Lifecycle of a task section below.

      Queue
       ATF uses AWS Simple Queue Service (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <lambda, priority>  pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is #lambdas x #priorities.

      Controller
       Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues. 

      The Controller serves tasks from its process local queue as a response to NextWork RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to NextWork RPCs.

      Executor
       The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop: 

      while True:
  w = get_next_work()
  do_work(w)

      Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.

      Heartbeat and Status Controller (HSC)
      The HSC serves RPCs for claiming a task for execution (ClaimTask), setting task status after execution (SetResults) and heartbeats during task execution (Heartbeat). ClaimTask requests originate from the Controllers in response to NextWork requests. Heartbeat and SetResults requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.

      Data model

      ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (assoc), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.

      Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.

      Lifecycle of a task

      1. Client performs a Schedule RPC call to Frontend with task information, including execution time. 
      2. Frontend creates Edgestore entity and assoc for the task. 
      3. When it is time to process the task, Store Consumer pulls the task from Edgestore and pushes it to a related SQS queue. 
      4. Executor makes NextWork RPC call to Controller, which pulls tasks from the SQS queue, makes a ClaimTask RPC to the HSC and then returns the task to the Executor. 
      5. Executor invokes the callback for the task. While processing, Executor performs Heartbeat RPC calls to Heartbeat and Status Controller (HSC). Once processing is done, Executor performs TaskStatus RPC call to HSC. 
      6. Upon getting Heartbeat and TaskStatus RPC calls, HSC updates the Edgestore entity and assoc.

      Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the assoc. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.

      Following are the task entity and association states in ATF and their corresponding timestamp updates:

      Entity status

      Assoc status

      next trigger timestamp in Assoc

      Comment

      new

      new

      scheduled_timestamp of the task

      Pick up new tasks that are ready. 

      enqueued

      started

      enqueued_timestamp + enqueue_timeout

      Re-enqueue task if it has been in enqueued state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.

      claimed

      started

      claimed_timestamp + claim_timeout

      Re-enqueue if task is claimed but never transfered to processing. This can happen if Controller is down after claiming a task. Task status is changed to enqueued after re-enqueue.

      processing

      started

      heartbeat_timestamp + heartbeat_timeout`

      Re-enqueue if task hasn’t sent heartbeat for too long. This can happen if Executor is down. Task status is changed to enqueued after re-enqueue. 

      retriable failure

      started

      compute next_timestamp according to backoff logic

      Exponential backoff for tasks with retriable failure. 

      success

      completed

      N/A

       

      fatal_failure

      completed

      N/A

       

      The store consumer polls for tasks based on the following query:

      assoc_status= && next_timestamp<=time.now()

      Below is the state machine that defines task state transitions: 

      Task State Transitions [Fig 2]

      Achieving guarantees

      At-least-once task execution
       At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a Success or a FatalFailure state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a RetriableFailure state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.

      No concurrent task execution
       Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (Claimed) before starting execution. Once the task execution is complete, the task status is updated to one of Success, FatalFailure or RetriableFailure. A task can be claimed only if its existing task state is Enqueued (retried tasks go to the Enqueued state as well once they are re-pushed onto SQS).

      However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution. 

      To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.

      Isolation
       Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.

      Delivery latency
       ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.

      Ownership model

      ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.

      Extending ATF

      As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF. 

      Periodic task execution
       Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.

      Better support for task chaining
       Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.

      Dead letter queues for misbehaving tasks
       One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.

      One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.

      Conclusion

      We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.
       

      // Tags
      // Copy link
      ================================================ FILE: test/test-pages/ebb-org/expected-metadata.json ================================================ { "title": "On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )", "byline": "Bradley M. Kuhn (http://ebb.org/bkuhn/)", "dir": null, "lang": "en-US", "excerpt": "The website of Bradley M. Kuhn, aka Brad, aka bkuhn. This site includes his GPG keys, resume, blog, projects list, software, interviews, speeches and writing.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/ebb-org/expected.html ================================================

      Tuesday 15 October 2019 by Bradley M. Kuhn

      The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to his resignation from the Free Software Foundation (FSF), to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities.

      I have been silent the last month because, until two days ago, I was an at-large member of FSF's Board of Directors, and a Voting Member of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis come directly from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request.

      The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the Board of Directors) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues.

      I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis.

      Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — seem to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me.

      For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed Selam G.'s Medium post raised an entirely new issue. In fact, RMS' views and statements posted on stallman.org about sexual morality escalated for the worse over the last few years. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which IMO didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma.

      IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to FOSS. (Indeed, I would not even be commenting on this issue if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with restraint, not with impunity.

      RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality.

      And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The FSF's mission is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission.

      I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with this “(re-)definition” of sexual assault. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is not a “frenzy”; it is a global movement by individuals who have been harmed seeking to hold both bad actors and society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the essay that I co-wrote with RMS and still find many of RMS' other essays compelling, important, and relevant.

      I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission.

      As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions.

      Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn.

      #include <std/disclaimer.h>
      use Standard::Disclaimer;
      from standard import disclaimer
      SELECT full_text FROM standard WHERE type = 'disclaimer';

      Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either.

      ebb ℠ is a service mark of Bradley M. Kuhn.

      ================================================ FILE: test/test-pages/ebb-org/source.html ================================================ On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )

      On Recent Controversial Events

      Tuesday 15 October 2019 by Bradley M. Kuhn

      The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to his resignation from the Free Software Foundation (FSF), to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities.

      I have been silent the last month because, until two days ago, I was an at-large member of FSF's Board of Directors, and a Voting Member of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis come directly from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request.

      The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the Board of Directors) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues.

      I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis.

      Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — seem to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me.

      For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed Selam G.'s Medium post raised an entirely new issue. In fact, RMS' views and statements posted on stallman.org about sexual morality escalated for the worse over the last few years. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which IMO didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma.

      IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to FOSS. (Indeed, I would not even be commenting on this issue if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with restraint, not with impunity.

      RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality.

      And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The FSF's mission is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission.

      I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with this “(re-)definition” of sexual assault. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is not a “frenzy”; it is a global movement by individuals who have been harmed seeking to hold both bad actors and society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the essay that I co-wrote with RMS and still find many of RMS' other essays compelling, important, and relevant.

      I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission.

      As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions.

      Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn.

      Submit comments on this post to <bkuhn@ebb.org>.


      Creative Commons License This website and all documents on it are licensed under a Creative Commons Attribution-Share Alike 3.0 United States License .

      #include <std/disclaimer.h>
      use Standard::Disclaimer;
      from standard import disclaimer
      SELECT full_text FROM standard WHERE type = 'disclaimer';

      Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either.

      — bkuhn

      ebb ℠ is a service mark of Bradley M. Kuhn.

      Bradley M. Kuhn <bkuhn@ebb.org>
      ================================================ FILE: test/test-pages/ehow-1/expected-metadata.json ================================================ { "title": "How to Build a Terrarium | eHow", "byline": "Lucy Akins", "dir": null, "lang": "en-US", "excerpt": "Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the...", "siteName": "eHow", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/ehow-1/expected.html ================================================

      Found This Helpful

      Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.

      Glass cloche terrariums
      Glass cloche terrariums (Lucy Akins)

      What You'll Need:

      • Cloche
      • Planter saucer, small shallow dish or desired platform
      • Floral foam oasis
      • Ruler
      • Spoon
      • Floral wire pins or paper clips
      • Small plants (from a florist or nursery)
      • Moss
      • Tweezers
      • Other small decorative items (optional)

      Step 1

      Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.

      Dig a hole in the oasis.
      Dig a hole in the oasis. (Lucy Akins)

      Step 2

      Insert your plant into the hole.

      Orchid in foam oasis
      Orchid in foam oasis (Lucy Akins)

      Step 3

      You can add various plants if you wish.

      Various foliage
      Various foliage (Lucy Akins)

      Step 4

      Using floral pins, attach enough moss around the oasis to cover it.

      Attach moss.
      Attach moss. (Lucy Akins)

      Step 5

      Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.

      Place cloche over oasis.
      Place cloche over oasis. (Lucy Akins)

      Step 6

      Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.

      Rearrange moss.
      Rearrange moss. (Lucy Akins)

      Step 7

      You can use any platform you wish. In this case, a small saucer was used.

      Place cloche on a platform to sit on.
      Place cloche on a platform to sit on. (Lucy Akins)

      Step 8

      This particular terrarium rests on a planter saucer and features a small white pumpkin.

      Cloche placed on a terracotta saucer
      Cloche placed on a terracotta saucer (Lucy Akins)

      Step 9

      This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.

      Placed on a wooden slice
      Placed on a wooden slice (Lucy Akins)

      Finished Terrarium

      Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.

      Cloche terrarium
      Cloche terrarium (Lucy Akins)

      Featured

      ================================================ FILE: test/test-pages/ehow-1/source.html ================================================ How to Build a Terrarium (with Pictures) | eHow

      How to Build a Terrarium

      Found This Helpful

      Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.

      Glass cloche terrariums
      Glass cloche terrariums (Lucy Akins)
      What You'll Need:
      • Cloche
      • Planter saucer, small shallow dish or desired platform
      • Floral foam oasis
      • Ruler
      • Spoon
      • Floral wire pins or paper clips
      • Small plants (from a florist or nursery)
      • Moss
      • Tweezers
      • Other small decorative items (optional)
      Step 1

      Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.

      Dig a hole in the oasis.
      Dig a hole in the oasis. (Lucy Akins)
      Step 2

      Insert your plant into the hole.

      Orchid in foam oasis
      Orchid in foam oasis (Lucy Akins)
      Step 3

      You can add various plants if you wish.

      Various foliage
      Various foliage (Lucy Akins)
      Step 4

      Using floral pins, attach enough moss around the oasis to cover it.

      Attach moss.
      Attach moss. (Lucy Akins)
      Step 5

      Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.

      Place cloche over oasis.
      Place cloche over oasis. (Lucy Akins)
      Step 6

      Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.

      Rearrange moss.
      Rearrange moss. (Lucy Akins)
      Step 7

      You can use any platform you wish. In this case, a small saucer was used.

      Place cloche on a platform to sit on.
      Place cloche on a platform to sit on. (Lucy Akins)
      Step 8

      This particular terrarium rests on a planter saucer and features a small white pumpkin.

      Cloche placed on a terracotta saucer
      Cloche placed on a terracotta saucer (Lucy Akins)
      Step 9

      This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.

      Placed on a wooden slice
      Placed on a wooden slice (Lucy Akins)
      Finished Terrarium

      Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.

      Cloche terrarium
      Cloche terrarium (Lucy Akins)
      Related Searches

      Featured

      Read Article
      How to Make a Ballet Tutu
      Read Article
      How to Paint a Tulip in Watercolor
      Read Article
      Kids Kitchen Slipcover

      Related Ads

      Related Searches
      Read Article

      How to Make Paper Sunflowers

      ================================================ FILE: test/test-pages/ehow-2/expected-metadata.json ================================================ { "title": "How to Throw a Graduation Party on a Budget | eHow", "byline": "Gina Roberts-Grey", "dir": null, "lang": "en-US", "excerpt": "Graduation parties are a great way to commemorate the years of hard work teens and college co-eds devote to education. They’re also costly for mom and dad.The average cost of a graduation party in 2013 was a whopping $1,200, according to Graduationparty.com; $700 of that was allocated for food. However that budget was based on Midwestern...", "siteName": "eHow", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/ehow-2/expected.html ================================================

      Save

      Graduation parties are a great way to commemorate the years of hard work teens and college co-eds devote to education. They’re also costly for mom and dad.

      The average cost of a graduation party in 2013 was a whopping $1,200, according to Graduationparty.com; $700 of that was allocated for food. However that budget was based on Midwestern statistics, and parties in urban areas like New York City are thought to have a much higher price tag.

      Thankfully, there are plenty of creative ways to trim a little grad party fat without sacrificing any of the fun or celebratory spirit.

      Graduation
      (Mike Watson Images/Moodboard/Getty)

      Parties hosted at restaurants, clubhouses and country clubs eliminate the need to spend hours cleaning up once party guests have gone home. But that convenience comes with a price tag. A country club may charge as much as $2,000 for room rental and restaurant food and beverage will almost always cost more than food prepped and served at home.

      Save money hosting the party at home.
      Thomas Jackson/Digital Vision/Getty Images

      Instead of hiring a DJ, use your iPod or Smartphone to spin the tunes. Both easily hook up to most speakers or mp3 compatible docks to play music from your music library. Or download Pandora, the free online radio app, and play hours of music for free.

      Personalize the music with a playlist of the grad’s favorite songs or songs that were big hits during his or her years in school.

      Online radio can take the place of a hired DJ.
      Spencer Platt/Getty Images News/Getty Images

      Avoid canned drinks, which guests often open, but don't finish. Serve pitchers of tap water with lemon and cucumber slices or sliced strawberries for an interesting and refreshing flavor. Opt for punches and non-alcoholic drinks for high school graduates that allow guests to dole out the exact amount they want to drink.

      Serve drinks in pitchers, not in cans.
      evgenyb/iStock/Getty Images

      Instead of inviting everyone you – and the graduate – know or ever knew, scale back the guest list. Forgo inviting guests that you or your grad haven't seen for eons. There is no reason to provide provisions for people who are essentially out of your lives. Sticking to a small, but personal, guest list allows more time to mingle with loved ones during the party, too.

      Limit guests to those close to the graduate.
      Kane Skennar/Photodisc/Getty Images

      See if your grad and his best friend, girlfriend or close family member would consider hosting a joint party. You can split some of the expenses, especially when the two graduates share mutual friends. You'll also have another parent to bounce ideas off of and to help you stick to your budget when you're tempted to splurge.

      Throw a joint bash for big savings.
      Mike Watson Images/Moodboard/Getty

      Skip carving stations of prime rib and jumbo shrimp as appetizers, especially for high school graduation parties. Instead, serve some of the graduate's favorite side dishes that are cost effective, like a big pot of spaghetti with breadsticks. Opt for easy and simple food such as pizza, finger food and mini appetizers.

      Avoid pre-packaged foods and pre-made deli platters. These can be quite costly. Instead, make your own cheese and deli platters for less than half the cost of pre-made.

      Cost effective appetizers are just as satisfying as pre-made deli platters.
      Mark Stout/iStock/Getty Images

      Instead of an evening dinner party, host a grad lunch or all appetizers party. Brunch and lunch fare or finger food costs less than dinner. Guests also tend to consume less alcohol in the middle of the day, which keeps cost down.

      A brunch gathering will cost less than a dinner party.
      Mark Stout/iStock/Getty Images

      Decorate your party in the graduate's current school colors or the colors of the school he or she will be headed to next. Décor that is not specifically graduation-themed may cost a bit less, and any leftovers can be re-used for future parties, picnics and events.

      Theme the party by color without graduation-specific decor.
      jethuynh/iStock/Getty Images

      Related Searches

      Promoted By Zergnet

      ================================================ FILE: test/test-pages/ehow-2/source.html ================================================ How to Throw a Graduation Party on a Budget (with Pictures) | eHow

      How to Throw a Graduation Party on a Budget

      eHow Contributor
      Save

      Graduation parties are a great way to commemorate the years of hard work teens and college co-eds devote to education. They’re also costly for mom and dad.

      The average cost of a graduation party in 2013 was a whopping $1,200, according to Graduationparty.com; $700 of that was allocated for food. However that budget was based on Midwestern statistics, and parties in urban areas like New York City are thought to have a much higher price tag.

      Thankfully, there are plenty of creative ways to trim a little grad party fat without sacrificing any of the fun or celebratory spirit.

      Graduation
      (Mike Watson Images/Moodboard/Getty)

      Parties hosted at restaurants, clubhouses and country clubs eliminate the need to spend hours cleaning up once party guests have gone home. But that convenience comes with a price tag. A country club may charge as much as $2,000 for room rental and restaurant food and beverage will almost always cost more than food prepped and served at home.
      Save money hosting the party at home.
      Thomas Jackson/Digital Vision/Getty Images

      Instead of hiring a DJ, use your iPod or Smartphone to spin the tunes. Both easily hook up to most speakers or mp3 compatible docks to play music from your music library. Or download Pandora, the free online radio app, and play hours of music for free.

      Personalize the music with a playlist of the grad’s favorite songs or songs that were big hits during his or her years in school.
      Online radio can take the place of a hired DJ.
      Spencer Platt/Getty Images News/Getty Images

      Avoid canned drinks, which guests often open, but don't finish. Serve pitchers of tap water with lemon and cucumber slices or sliced strawberries for an interesting and refreshing flavor. Opt for punches and non-alcoholic drinks for high school graduates that allow guests to dole out the exact amount they want to drink.
      Serve drinks in pitchers, not in cans.
      evgenyb/iStock/Getty Images

      Instead of inviting everyone you – and the graduate – know or ever knew, scale back the guest list. Forgo inviting guests that you or your grad haven't seen for eons. There is no reason to provide provisions for people who are essentially out of your lives. Sticking to a small, but personal, guest list allows more time to mingle with loved ones during the party, too.
      Limit guests to those close to the graduate.
      Kane Skennar/Photodisc/Getty Images

      See if your grad and his best friend, girlfriend or close family member would consider hosting a joint party. You can split some of the expenses, especially when the two graduates share mutual friends. You'll also have another parent to bounce ideas off of and to help you stick to your budget when you're tempted to splurge.
      Throw a joint bash for big savings.
      Mike Watson Images/Moodboard/Getty

      Skip carving stations of prime rib and jumbo shrimp as appetizers, especially for high school graduation parties. Instead, serve some of the graduate's favorite side dishes that are cost effective, like a big pot of spaghetti with breadsticks. Opt for easy and simple food such as pizza, finger food and mini appetizers.

      Avoid pre-packaged foods and pre-made deli platters. These can be quite costly. Instead, make your own cheese and deli platters for less than half the cost of pre-made.
      Cost effective appetizers are just as satisfying as pre-made deli platters.
      Mark Stout/iStock/Getty Images

      Instead of an evening dinner party, host a grad lunch or all appetizers party. Brunch and lunch fare or finger food costs less than dinner. Guests also tend to consume less alcohol in the middle of the day, which keeps cost down.
      A brunch gathering will cost less than a dinner party.
      Mark Stout/iStock/Getty Images

      Decorate your party in the graduate's current school colors or the colors of the school he or she will be headed to next. Décor that is not specifically graduation-themed may cost a bit less, and any leftovers can be re-used for future parties, picnics and events.
      Theme the party by color without graduation-specific decor.
      jethuynh/iStock/Getty Images

      Related Searches

      Promoted By Zergnet

      Comments

      Related Searches

      Check It Out

      13 Delicious Thanksgiving Sides That'll Make Turkey Insignificant

      M
      Is DIY in your DNA? Become part of our maker community.
      Submit Your Work!
      ================================================ FILE: test/test-pages/embedded-videos/expected-metadata.json ================================================ { "title": "Embedded videos test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/embedded-videos/expected.html ================================================

      Lorem

      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      At root

      In a paragraph

      In a div

      Foo

      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      ================================================ FILE: test/test-pages/embedded-videos/source.html ================================================ Embedded videos test

      Lorem

      Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

      Videos

      At root

      In a paragraph

      In a div

      Foo

      Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
      ================================================ FILE: test/test-pages/engadget/expected-metadata.json ================================================ { "title": "Xbox One X review: A console that keeps up with gaming PCs", "byline": "Devindra Hardawar", "dir": null, "lang": "en", "excerpt": "The Xbox One X is the most powerful gaming console ever, but it's not for everyone yet.", "siteName": "Engadget", "publishedTime": "2017-11-03 03:01:00.000000", "readerable": true } ================================================ FILE: test/test-pages/engadget/expected.html ================================================

      The Xbox One X is the ultimate video game system. It sports more horsepower than any system ever. And it plays more titles in native 4K than Sony's PlayStation 4 Pro. It's just about everything you could want without investing in a gaming PC. The only problem? It's now been a year since the PS4 Pro launched, and the One X costs $500, while Sony's console launched at $400. That high price limits the Xbox One X to diehard Microsoft fans who don't mind paying a bit more to play the console's exclusive titles in 4K. Everyone else might be better off waiting, or opting for the $279 Xbox One S.

      Gallery: Xbox One X | 14 Photos

      from $610.00

      87

      • Most powerful hardware ever in a home console
      • Solid selection of enhanced titles
      • 4K Blu-ray drive is great for movie fans
      • Expensive
      • Not worth it if you don’t have a 4K TV
      • Still no VR support

      As promised, the Xbox One X is the most powerful game console ever. In practice, though, it really just puts Microsoft on equal footing with Sony’s PlayStation 4 Pro. 4K/HDR enhanced games look great, but it’s lack of VR is disappointing in 2017.

      Hardware

      Despite all the power inside, the One X is Microsoft's smallest console to date. It looks similar to the Xbox One S, except it has an entirely matte black case and is slightly slimmer. It's also surprisingly dense -- the console weighs 8.4 pounds, but it feels far heavier than you'd expect for its size, thanks to all of its new hardware. The One S, in comparison, weighs two pounds less.

      The Xbox One X's real upgrades are under the hood. It features an 8-core CPU running at 2.3Ghz, 12GB of GDDR5 RAM, a 1 terabyte hard drive and an upgraded AMD Polaris GPU with 6 teraflops of computing power. The PS4 Pro has only 8GB of RAM and tops out at 4.2 teraflops. Microsoft's console is clearly faster. That additional horsepower means the Xbox One X can run more games in full native 4K than the Sony's console.

      Along the front, there's the slot-loading 4K Blu-ray drive, a physical power button, a single USB port and a controller pairing button. And around back, there are HDMI out and in ports, the latter of which lets you plug in your cable box. Additionally, there are two USB ports, connections for optical audio, IR out, and gigabit Ethernet. If you've still got a Kinect around, you'll need to use a USB adapter to plug it in.

      Devindra Hardawar/AOL

      The console's controller hasn't changed since its last mini-upgrade with the Xbox One S. That revision rounded out its seams, improved bumper performance and added a 3.5mm headphone jack. It's still a great controller, though I'm annoyed Microsoft is sticking with AA batteries as their default power source. Sure, you could just pick up some renewable batteries, or the Play and Charge kit, but that's an extra expense. And manually swapping batteries feels like a bad user experience when every other console has rechargeable controllers.

      In use

      Devindra Hardawar/AOL

      You won't find any major differences between the One X and the last Xbox at first — aside from a more dramatic startup sequence. Navigating the Xbox interface is fast and zippy, but mostly that's due to a recent OS upgrade. If you're moving over from an older Xbox One, you can use the backup tool to transfer your games and settings to an external hard drive. Just plug that into the new console during setup and it'll make it feel just like your old machine. It's also a lot faster than waiting for everything to download from Xbox Live.

      You'll still have to set aside some time if you want to play an Xbox One X-enhanced title, though. Those 4K textures will make games significantly larger, but Microsoft says it's come up with a few ways to help developers make downloading them more efficient. For example, language packs and other optional content won't get installed by default.

      We only had a few enhanced titles to test out during our review: Gears of War 4, Killer Instinct and Super Lucky's Tale. They each took advantage of the console in different ways. Gears of War 4 runs natively in 4K at 30 FPS with Dolby Atmos and HDR (high dynamic range lighting) support. It looked great -- especially with HDR, which highlighted bright elements like lightning strikes -- but I noticed the frame rate dip occasionally. I was also surprised that load times were on-par with what I've seen with the game on the Xbox One S.

      You can also play in Performance mode, which bumps the frame rate up to 60FPS and uses higher quality graphical effects, while rendering it lower in 1080p. Personally, I preferred this, since it makes the game much smoother -- as if you're playing it on a high-end gaming PC, not a console. Some PlayStation 4 Pro games also let you choose how you wanted to distribute its power, so in some ways Microsoft is just following in its footsteps.

      I've been playing Gears of War 4 on my gaming PC (which is connected to my home theater) over the past year, and I was impressed that the Xbox One X is able to deliver a similar experience. It didn't quite match my rig though, which is powered by Intel Core i7 4790k CPU running at 4GHz, 16GB DDR3 RAM and an NVIDIA GTX 1080 GPU. Typically, I play at 1,440p (2,560 by 1,440 pixels) with HDR and all of the graphical settings set to their highest level, and I can easily maintain a 60FPS frame rate. The One X felt just as solid at 1080p, but there were clearly plenty of graphics settings it couldn't take advantage of, in particular higher levels of bloom lighting and shadow detail.

      Gallery: Xbox One X screenshots | 9 Photos

      Killer Instinct and Super Lucky's Tale run in 4K at a smooth 60FPS. They both looked and played better than their standard versions, though I was surprised they didn't take advantage of HDR. As usual, I noticed the improvement in frame rates more than the higher resolution. Unless you're sitting very close to a TV above 50-inches, you'd likely have a hard time telling between 4K and 1080p.

      That poses a problem for Microsoft: It's betting that gamers will actually want true 4K rendering. In practice, though, PlayStation 4 Pro titles running in HDR and resolutions between 1080p and 4K often look just as good to the naked eye. The Xbox One X's big advantage is that its hardware could let more games reach 60FPS compared to Sony's console.

      Microsoft says over 130 Xbox One X-enhanced titles are in the works. That includes already-released games like Forza Motorsport 7 and Assassin's Creed Origins, as well as upcoming titles like Call of Duty: WW2. You'll be able to find them easily in a special section in the Xbox store. There is also a handful of Xbox 360 games that'll get enhanced eventually, including Halo 3 and Fallout 3. Some of those titles will get bumped up to a higher resolution, while others will get HDR support. Microsoft describes these upgrades as a bonus for developers who were prescient about how they built their games. Basically, don't expect your entire 360 library to get enhanced.

      Even if a game isn't specifically tuned for the new console, Microsoft says you might still see some performance improvements. The PlayStation 4 Pro, meanwhile, has over one hundred games built for its hardware, and its boost mode can speed up some older games.

      Microsoft is still pushing the Xbox as more than just a game console, though. 4K Blu-rays loaded up quickly, and I didn't notice many delays as I skipped around films. Planet Earth II, in particular, looked fantastic thanks to its brilliant use of HDR. Unfortunately, the One X doesn't support Dolby Vision, so you're stuck with the slightly less capable HDR 10 standard. That makes sense since it's more widely supported, but it would have been nice to see Dolby's, too.

      And speaking of Dolby technology, Microsoft is also highlighting Atmos support on the One X, just like it did with the One S. The company's app lets you configure the console to pass audio Atmos signals to your audio receiver. You can also shell out $15 to get Atmos support for headphones, which simulates immersive surround sound. It's strange to pay money to unlock Dolby features, but it's worth it since it's significantly better than Microsoft's audio virtualization technology. The Netflix app also supports Atmos for a handful of films (something that the Xbox One S and PlayStation 4 offer, as well).

      One thing you won't find in the new Xbox is VR support. Microsoft has mentioned that the console will offer some sort of mixed reality, but it hasn't offered up any details yet. It's technically powerful enough to work with any of the Windows Mixed Reality headsets launching this fall. It's a shame that Microsoft is being so wishy-washy because Sony has had a very successful head start with the PlayStation VR.

      Pricing and the competition

      Devindra Hardawar/AOL

      The biggest knock against the Xbox One X is its $500 price. The PS4 Pro launched at $400 last year, and there's a good chance we'll see plenty of deals around the holidays. If your friends are on Xbox Live, or you're a devotee of Microsoft's first party franchises, then the X makes more sense. If you just want to play third-party titles that come to both platforms, though, the PS4 Pro is clearly the better deal.

      If you're looking to upgrade from an original Xbox One, and you have a new TV, the One X might be more compelling. It's faster and offers more features than the One S, and more importantly, it'll last you much longer without needing an upgrade. There's also plenty of wisdom in simply waiting a while before you buy the One X, especially if you haven't moved to a 4K TV yet. The new console can make games look better on 1080p sets, since it'll supersample high-res textures and have more graphical effects, but it's simply not worth the upgrade since those TVs don't support HDR.

      If price isn't a huge concern for you, it's worth considering investing in a gaming PC. A decent one costs between $600 and $800, plus the price of a monitor, but it'll easily be more powerful than the One X. And you have the added benefit of upgrading components down the line. Now that Microsoft and game publishers are offering most major titles on PC, you won't be missing out on much by ditching consoles.

      Wrap-up

      Ultimately, the Xbox One X offers some major performance upgrades that gamers will notice -- especially if you're coming from an original Xbox One. But it's also a bit disappointing since it's coming a year after the PS4 Pro, and it doesn't offer VR yet. For Microsoft fans, though, none of that will matter. It's exactly what the company promised: the fastest game console ever made.

      ================================================ FILE: test/test-pages/engadget/source.html ================================================ Xbox One X review: A console that keeps up with gaming PCs
      Show More Results

      Xbox One X review: A console that keeps up with gaming PCs

      But only hardcore gamers will appreciate it.

      Devindra Hardawar, @devindra
      11.03.17 in Gadgetry
      Comments
      2192 Shares
      Share
      Tweet
      Share
      Save

        The Xbox One X is the ultimate video game system. It sports more horsepower than any system ever. And it plays more titles in native 4K than Sony's PlayStation 4 Pro. It's just about everything you could want without investing in a gaming PC. The only problem? It's now been a year since the PS4 Pro launched, and the One X costs $500, while Sony's console launched at $400. That high price limits the Xbox One X to diehard Microsoft fans who don't mind paying a bit more to play the console's exclusive titles in 4K. Everyone else might be better off waiting, or opting for the $279 Xbox One S.

        Gallery: Xbox One X | 14 Photos

        14

        Engadget Score
        Poor
        Uninspiring
        Good
        Excellent
        Key

        Microsoft
        Xbox
        from $610.00
        Buy Now
        87
        Pros
        • Most powerful hardware ever in a home console
        • Solid selection of enhanced titles
        • 4K Blu-ray drive is great for movie fans
        Cons
        • Expensive
        • Not worth it if you don’t have a 4K TV
        • Still no VR support

        Summary

        As promised, the Xbox One X is the most powerful game console ever. In practice, though, it really just puts Microsoft on equal footing with Sony’s PlayStation 4 Pro. 4K/HDR enhanced games look great, but it’s lack of VR is disappointing in 2017.

        Hardware

        Despite all the power inside, the One X is Microsoft's smallest console to date. It looks similar to the Xbox One S, except it has an entirely matte black case and is slightly slimmer. It's also surprisingly dense -- the console weighs 8.4 pounds, but it feels far heavier than you'd expect for its size, thanks to all of its new hardware. The One S, in comparison, weighs two pounds less.

        The Xbox One X's real upgrades are under the hood. It features an 8-core CPU running at 2.3Ghz, 12GB of GDDR5 RAM, a 1 terabyte hard drive and an upgraded AMD Polaris GPU with 6 teraflops of computing power. The PS4 Pro has only 8GB of RAM and tops out at 4.2 teraflops. Microsoft's console is clearly faster. That additional horsepower means the Xbox One X can run more games in full native 4K than the Sony's console.

        Along the front, there's the slot-loading 4K Blu-ray drive, a physical power button, a single USB port and a controller pairing button. And around back, there are HDMI out and in ports, the latter of which lets you plug in your cable box. Additionally, there are two USB ports, connections for optical audio, IR out, and gigabit Ethernet. If you've still got a Kinect around, you'll need to use a USB adapter to plug it in.

        Devindra Hardawar/AOL

        The console's controller hasn't changed since its last mini-upgrade with the Xbox One S. That revision rounded out its seams, improved bumper performance and added a 3.5mm headphone jack. It's still a great controller, though I'm annoyed Microsoft is sticking with AA batteries as their default power source. Sure, you could just pick up some renewable batteries, or the Play and Charge kit, but that's an extra expense. And manually swapping batteries feels like a bad user experience when every other console has rechargeable controllers.

        In use

        Devindra Hardawar/AOL

        You won't find any major differences between the One X and the last Xbox at first — aside from a more dramatic startup sequence. Navigating the Xbox interface is fast and zippy, but mostly that's due to a recent OS upgrade. If you're moving over from an older Xbox One, you can use the backup tool to transfer your games and settings to an external hard drive. Just plug that into the new console during setup and it'll make it feel just like your old machine. It's also a lot faster than waiting for everything to download from Xbox Live.

        You'll still have to set aside some time if you want to play an Xbox One X-enhanced title, though. Those 4K textures will make games significantly larger, but Microsoft says it's come up with a few ways to help developers make downloading them more efficient. For example, language packs and other optional content won't get installed by default.

        We only had a few enhanced titles to test out during our review: Gears of War 4, Killer Instinct and Super Lucky's Tale. They each took advantage of the console in different ways. Gears of War 4 runs natively in 4K at 30 FPS with Dolby Atmos and HDR (high dynamic range lighting) support. It looked great -- especially with HDR, which highlighted bright elements like lightning strikes -- but I noticed the frame rate dip occasionally. I was also surprised that load times were on-par with what I've seen with the game on the Xbox One S.

        You can also play in Performance mode, which bumps the frame rate up to 60FPS and uses higher quality graphical effects, while rendering it lower in 1080p. Personally, I preferred this, since it makes the game much smoother -- as if you're playing it on a high-end gaming PC, not a console. Some PlayStation 4 Pro games also let you choose how you wanted to distribute its power, so in some ways Microsoft is just following in its footsteps.

        I've been playing Gears of War 4 on my gaming PC (which is connected to my home theater) over the past year, and I was impressed that the Xbox One X is able to deliver a similar experience. It didn't quite match my rig though, which is powered by Intel Core i7 4790k CPU running at 4GHz, 16GB DDR3 RAM and an NVIDIA GTX 1080 GPU. Typically, I play at 1,440p (2,560 by 1,440 pixels) with HDR and all of the graphical settings set to their highest level, and I can easily maintain a 60FPS frame rate. The One X felt just as solid at 1080p, but there were clearly plenty of graphics settings it couldn't take advantage of, in particular higher levels of bloom lighting and shadow detail.

        Gallery: Xbox One X screenshots | 9 Photos

        9

        Killer Instinct and Super Lucky's Tale run in 4K at a smooth 60FPS. They both looked and played better than their standard versions, though I was surprised they didn't take advantage of HDR. As usual, I noticed the improvement in frame rates more than the higher resolution. Unless you're sitting very close to a TV above 50-inches, you'd likely have a hard time telling between 4K and 1080p.

        That poses a problem for Microsoft: It's betting that gamers will actually want true 4K rendering. In practice, though, PlayStation 4 Pro titles running in HDR and resolutions between 1080p and 4K often look just as good to the naked eye. The Xbox One X's big advantage is that its hardware could let more games reach 60FPS compared to Sony's console.

        Microsoft says over 130 Xbox One X-enhanced titles are in the works. That includes already-released games like Forza Motorsport 7 and Assassin's Creed Origins, as well as upcoming titles like Call of Duty: WW2. You'll be able to find them easily in a special section in the Xbox store. There is also a handful of Xbox 360 games that'll get enhanced eventually, including Halo 3 and Fallout 3. Some of those titles will get bumped up to a higher resolution, while others will get HDR support. Microsoft describes these upgrades as a bonus for developers who were prescient about how they built their games. Basically, don't expect your entire 360 library to get enhanced.

        Even if a game isn't specifically tuned for the new console, Microsoft says you might still see some performance improvements. The PlayStation 4 Pro, meanwhile, has over one hundred games built for its hardware, and its boost mode can speed up some older games.

        Microsoft is still pushing the Xbox as more than just a game console, though. 4K Blu-rays loaded up quickly, and I didn't notice many delays as I skipped around films. Planet Earth II, in particular, looked fantastic thanks to its brilliant use of HDR. Unfortunately, the One X doesn't support Dolby Vision, so you're stuck with the slightly less capable HDR 10 standard. That makes sense since it's more widely supported, but it would have been nice to see Dolby's, too.

        And speaking of Dolby technology, Microsoft is also highlighting Atmos support on the One X, just like it did with the One S. The company's app lets you configure the console to pass audio Atmos signals to your audio receiver. You can also shell out $15 to get Atmos support for headphones, which simulates immersive surround sound. It's strange to pay money to unlock Dolby features, but it's worth it since it's significantly better than Microsoft's audio virtualization technology. The Netflix app also supports Atmos for a handful of films (something that the Xbox One S and PlayStation 4 offer, as well).

        One thing you won't find in the new Xbox is VR support. Microsoft has mentioned that the console will offer some sort of mixed reality, but it hasn't offered up any details yet. It's technically powerful enough to work with any of the Windows Mixed Reality headsets launching this fall. It's a shame that Microsoft is being so wishy-washy because Sony has had a very successful head start with the PlayStation VR.

        Pricing and the competition

        Devindra Hardawar/AOL

        The biggest knock against the Xbox One X is its $500 price. The PS4 Pro launched at $400 last year, and there's a good chance we'll see plenty of deals around the holidays. If your friends are on Xbox Live, or you're a devotee of Microsoft's first party franchises, then the X makes more sense. If you just want to play third-party titles that come to both platforms, though, the PS4 Pro is clearly the better deal.

        If you're looking to upgrade from an original Xbox One, and you have a new TV, the One X might be more compelling. It's faster and offers more features than the One S, and more importantly, it'll last you much longer without needing an upgrade. There's also plenty of wisdom in simply waiting a while before you buy the One X, especially if you haven't moved to a 4K TV yet. The new console can make games look better on 1080p sets, since it'll supersample high-res textures and have more graphical effects, but it's simply not worth the upgrade since those TVs don't support HDR.

        If price isn't a huge concern for you, it's worth considering investing in a gaming PC. A decent one costs between $600 and $800, plus the price of a monitor, but it'll easily be more powerful than the One X. And you have the added benefit of upgrading components down the line. Now that Microsoft and game publishers are offering most major titles on PC, you won't be missing out on much by ditching consoles.

        Wrap-up

        Ultimately, the Xbox One X offers some major performance upgrades that gamers will notice -- especially if you're coming from an original Xbox One. But it's also a bit disappointing since it's coming a year after the PS4 Pro, and it doesn't offer VR yet. For Microsoft fans, though, none of that will matter. It's exactly what the company promised: the fastest game console ever made.

        From around the web

        ear icon eye icon text file vr
        ================================================ FILE: test/test-pages/firefox-nightly-blog/expected-metadata.json ================================================ { "title": "These Weeks in Firefox: Issue 85 – Firefox Nightly News", "byline": "Mike Conley", "dir": "ltr", "lang": "en-US", "excerpt": "Highlights Here's our Firefox Year in Review! Here’s our Performance Year in Review! We've just landed Bug 1553982, which aims to prevent starting an update while another Firefox instance ...", "siteName": "Firefox Nightly News", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/firefox-nightly-blog/expected.html ================================================

        Highlights

        • Here’s our Firefox Year in Review!
        • Here’s our Performance Year in Review!
        • We’ve just landed Bug 1553982, which aims to prevent starting an update while another Firefox instance is running (the cause of that about:restartrequired error page you may have seen).

          • The about:restartrequired error page, saying "Sorry. We just need to do one small thing to keep going. Nightly has just been updated in the background. Click Restart Nightly to complete the update. We will restore all your pages, windows and tabs afterwards, so you can be on your way quickly.", followed by a button to restart Nightly.

            Users who run multiple user profiles concurrently will probably see this less!

        • Also just about to land is Bug 353804, which provides some support for downloading new updates when we already have an update downloaded but haven’t installed it yet. That should prevent many cases of restarting to finish an update and then immediately being notified about another one.
        • Thanks to evilpie, users can now import logins from Keepass(XC) into Firefox
        • From Firefox 85 it’s now possible to disable tab-to-search on a per-engine basis, by unchecking a search engine in Search Preferences. That will both hide the shortcut button and disable tab-to-search for the engine. (Bug 1681512)
        • From Firefox 85 it’s also possible to disable tab-to-search globally by unchecking the Search Engines checkbox in the Address Bar Preferences, under Privacy & Security.
        • Firefox now supports printing non-contiguous page ranges (e.g. 1-3, 6, 7) – Bug 499640
        • DevTools and Marionette are now fully Fission compatible! Congratulations to those teams!
          • Reminder: Nightly users can help us test Fission by enabling it in about:preferences#experimental, and filing bugs here

        Friends of the Firefox team

        Introductions/Shout-Outs

        • [harry] Amy Churchwell joins the Search & Navigation team today. She transferred internally from Marketing Engineering. Welcome Amy!

        Resolved bugs (excluding employees)

        Fixed more than one bug

        • Masatoshi Kimura [:emk]
        • Michelle Goossens [:masterwayz]
        • Sonia
        • Tim Nguyen :ntim

        New contributors (🌟 = first patch)

        Project Updates

        Add-ons / Web Extensions

        Addon Manager & about:addons
        • Starting from Firefox 85, Mozilla-signed privileged addons can be installed from a third party website without triggering the “third party addon install doorhanger” (and without having to add new “install” site permission for those hosts, e.g. as we had to do for fpn.firefox.com) – Bug 1681331
        WebExtensions Framework
        • Ankush Dua contributed a fix for the devtools optional_permission (the devtools optional_permission can be used by extension, like ABP, that provides a devtools panel as a secondary feature of the addon) – Bug 1671579
        • Fixed content scripts applied to webpages loaded as subframes of an extension browserAction/pageAction popup when Fission is enabled – Bug 1680877
        • Fixed addon startup issue when webRequest is moved from permissions to optional_permissions in an addon update (regression from Bug 1624235) – Bug 1637059

        Developer Tools

        • DevTools Fission M2 – Making DevTools Fission compatible DONE.

          • A table showing the total number of remaining bugs for the MVP to make the DevTools Fission-compatible.

            Our DevTools are ready for Fission (out-of-process iframes)!

        • Marionette Fission – Making Marionette Fission compatible DONE

          • A table showing the total number of remaining bugs for the MVP to make Marionette Fission-compatible.

            Marionette, the framework that allows Firefox to be tested with automation, is now Fission compatible too!

        Fission

        Installer & Updater

        • Background updater work is also proceeding, with Bug 1676296 landing last week to support managing scheduled tasks in Gecko, and more development still also happening on the background task framework.

        New Tab Page and Pocket

        • We’re running three experiments:
          • Newtab Pocket stories in AU and NZ
          • New signup/login call-to-action in the Pocket doorhanger
          • We’re testing some changes to newtab story personalization

        Password Manager

        • Dimi fixed Bug 1677710 The password manager code triggers main thread sqlite disk I/O off of the gather-telemetry notification
        • And Bug 1678200 Remove or update probes expiring in Firefox 86: pwmgr.doorhanger_submitted#doorhanger_submitted
        • Thanks for Kenrick95 for fixing Bug 1678616 about:logins menu problem
        • 2021 Planning underway

        PDFs & Printing

        • mstriemer put a Printing… message in the dialog and hid the popup dialog which showed progress, the cancel button on that dialog caused problems and it looked dated Bug 1679133
        • mstriemer hid the print setting that don’t relate to PDFs when a PDF is being printed Bug 1669725
        • mstriemer updated the form to be disabled when loading a printer’s settings. Sometimes loading a physical printer’s settings can take a few settings and changes could be lost in this time Bug 1676388
        • emalysz made a change to avoid updating the preview for some settings that can’t change the preview output Bug 1676199
        • sfoster added a paginator to the preview when it’s hovered to show current page, next/prev/first/last buttons Bug 1654684
        • emalysz added support for non-contiguous page ranges (ex: 1-3, 6, 7) Bug 499640
        • emalysz fixed an issue where the form could get disabled with custom margins interactions Bug 1674106

        Performance

        Picture-in-Picture

        Search and Navigation

        • Fixed regressions related to Input Method Editor, in particular loss of the last token (Bug 1673669) and race conditions causing the wrong search engine to be used or Search Mode to be lost (Bug 1679697, Bug 1678647)
        • Introduced a new advanced preference to keep the Address Bar results panel open during IME composition. This provides a better experience for keyboard layouts that don’t open a picker panel. In the future we hope to be able to auto-detect that situation, but in the meanwhile, you can flip browser.urlbar.imeCompositionClosesPanel to false and test the alternative behavior (Bug 1673971)
        • URL canonization (www.*.com) now uses https by default, the protocol can be customized through the browser.fixup.alternate.protocol advanced pref (Bug 1638215)
        • Work continued on the weather QuickSuggest experiment, but its release has been moved to January.
        • Region.jsm now can use a Geolocation monitor to update without hitting the network (Bug 1663501)
        • Fixed a bug where search engines were being re-added on startup after their removal, when using a language pack (Bug 1675624)

        2 comments on “These Weeks in Firefox: Issue 85”

        Post a comment

        1. Alan Goodale wrote on

          Enhancement: Are there plans to provide a “collections” functionality for the desktop similar to what was released on Android? I need to be able to group the current instance of tabs into common “themes”. Close and open entire collections (themes) independently, with a single click. Read and update those same tabs, and then close them again with the latest URL of each tab being saved, rather than just the original seed URL. It’s similar to the “Restore previous session” functionality currently under Options\General\Startup, but different in that there would be multiple named “sessions” that could be restored individually, rather than just the one. It would also be helpful if the original seed URL for each tab could still be accessible, perhaps through a right click on the tab, but the latest viewed instance is the most important. Sync to Android would be a very desirable bonus.

          Reply

        2. John Doe wrote on

          Thank you for fixing Bug 1553982! I have moved most of my work to another browser due to the need to restart so often but will try Firefox again now.

          Reply

        ================================================ FILE: test/test-pages/firefox-nightly-blog/source.html ================================================ These Weeks in Firefox: Issue 85 – Firefox Nightly News

        Firefox Nightly News

        Let's improve quality, build after build!

        Categories: News

        These Weeks in Firefox: Issue 85

        2 responses

        Highlights

        • Here’s our Firefox Year in Review!
        • Here’s our Performance Year in Review!
        • We’ve just landed Bug 1553982, which aims to prevent starting an update while another Firefox instance is running (the cause of that about:restartrequired error page you may have seen).
          • The about:restartrequired error page, saying "Sorry. We just need to do one small thing to keep going. Nightly has just been updated in the background. Click Restart Nightly to complete the update. We will restore all your pages, windows and tabs afterwards, so you can be on your way quickly.", followed by a button to restart Nightly.

            Users who run multiple user profiles concurrently will probably see this less!

        • Also just about to land is Bug 353804, which provides some support for downloading new updates when we already have an update downloaded but haven’t installed it yet. That should prevent many cases of restarting to finish an update and then immediately being notified about another one.
        • Thanks to evilpie, users can now import logins from Keepass(XC) into Firefox
        • From Firefox 85 it’s now possible to disable tab-to-search on a per-engine basis, by unchecking a search engine in Search Preferences. That will both hide the shortcut button and disable tab-to-search for the engine. (Bug 1681512)
        • From Firefox 85 it’s also possible to disable tab-to-search globally by unchecking the Search Engines checkbox in the Address Bar Preferences, under Privacy & Security.
        • Firefox now supports printing non-contiguous page ranges (e.g. 1-3, 6, 7) – Bug 499640
        • DevTools and Marionette are now fully Fission compatible! Congratulations to those teams!
          • Reminder: Nightly users can help us test Fission by enabling it in about:preferences#experimental, and filing bugs here

        Friends of the Firefox team

        Introductions/Shout-Outs

        • [harry] Amy Churchwell joins the Search & Navigation team today. She transferred internally from Marketing Engineering. Welcome Amy!

        Resolved bugs (excluding employees)

        Fixed more than one bug

        • Masatoshi Kimura [:emk]
        • Michelle Goossens [:masterwayz]
        • Sonia
        • Tim Nguyen :ntim

        New contributors (🌟 = first patch)

        Project Updates

        Add-ons / Web Extensions

        Addon Manager & about:addons
        • Starting from Firefox 85, Mozilla-signed privileged addons can be installed from a third party website without triggering the “third party addon install doorhanger” (and without having to add new “install” site permission for those hosts, e.g. as we had to do for fpn.firefox.com) – Bug 1681331

         

        WebExtensions Framework
        • Ankush Dua contributed a fix for the devtools optional_permission (the devtools optional_permission can be used by extension, like ABP, that provides a devtools panel as a secondary feature of the addon) – Bug 1671579
        • Fixed content scripts applied to webpages loaded as subframes of an extension browserAction/pageAction popup when Fission is enabled – Bug 1680877
        • Fixed addon startup issue when webRequest is moved from permissions to optional_permissions in an addon update (regression from Bug 1624235) – Bug 1637059

        Developer Tools

        • DevTools Fission M2 – Making DevTools Fission compatible DONE.
          • A table showing the total number of remaining bugs for the MVP to make the DevTools Fission-compatible.

            Our DevTools are ready for Fission (out-of-process iframes)!

        • Marionette Fission – Making Marionette Fission compatible DONE
          • A table showing the total number of remaining bugs for the MVP to make Marionette Fission-compatible.

            Marionette, the framework that allows Firefox to be tested with automation, is now Fission compatible too!

        Fission

        Installer & Updater

        • Background updater work is also proceeding, with Bug 1676296 landing last week to support managing scheduled tasks in Gecko, and more development still also happening on the background task framework.

        New Tab Page and Pocket

        • We’re running three experiments:
          • Newtab Pocket stories in AU and NZ
          • New signup/login call-to-action in the Pocket doorhanger
          • We’re testing some changes to newtab story personalization

        Password Manager

        • Dimi fixed Bug 1677710 The password manager code triggers main thread sqlite disk I/O off of the gather-telemetry notification
        • And Bug 1678200 Remove or update probes expiring in Firefox 86: pwmgr.doorhanger_submitted#doorhanger_submitted
        • Thanks for Kenrick95 for fixing Bug 1678616 about:logins menu problem
        • 2021 Planning underway

        PDFs & Printing

        • mstriemer put a Printing… message in the dialog and hid the popup dialog which showed progress, the cancel button on that dialog caused problems and it looked dated Bug 1679133
        • mstriemer hid the print setting that don’t relate to PDFs when a PDF is being printed Bug 1669725
        • mstriemer updated the form to be disabled when loading a printer’s settings. Sometimes loading a physical printer’s settings can take a few settings and changes could be lost in this time Bug 1676388
        • emalysz made a change to avoid updating the preview for some settings that can’t change the preview output Bug 1676199
        • sfoster added a paginator to the preview when it’s hovered to show current page, next/prev/first/last buttons Bug 1654684
        • emalysz added support for non-contiguous page ranges (ex: 1-3, 6, 7) Bug 499640
        • emalysz fixed an issue where the form could get disabled with custom margins interactions Bug 1674106

        Performance

        Picture-in-Picture

        Search and Navigation

        • Fixed regressions related to Input Method Editor, in particular loss of the last token (Bug 1673669) and race conditions causing the wrong search engine to be used or Search Mode to be lost (Bug 1679697, Bug 1678647)
        • Introduced a new advanced preference to keep the Address Bar results panel open during IME composition. This provides a better experience for keyboard layouts that don’t open a picker panel. In the future we hope to be able to auto-detect that situation, but in the meanwhile, you can flip browser.urlbar.imeCompositionClosesPanel to false and test the alternative behavior (Bug 1673971)
        • URL canonization (www.*.com) now uses https by default, the protocol can be customized through the browser.fixup.alternate.protocol advanced pref (Bug 1638215)
        • Work continued on the weather QuickSuggest experiment, but its release has been moved to January.
        • Region.jsm now can use a Geolocation monitor to update without hitting the network (Bug 1663501)
        • Fixed a bug where search engines were being re-added on startup after their removal, when using a language pack (Bug 1675624)

        2 comments on “These Weeks in Firefox: Issue 85”

        Post a comment

        1. Alan Goodale wrote on

          Enhancement: Are there plans to provide a “collections” functionality for the desktop similar to what was released on Android? I need to be able to group the current instance of tabs into common “themes”. Close and open entire collections (themes) independently, with a single click. Read and update those same tabs, and then close them again with the latest URL of each tab being saved, rather than just the original seed URL. It’s similar to the “Restore previous session” functionality currently under Options\General\Startup, but different in that there would be multiple named “sessions” that could be restored individually, rather than just the one. It would also be helpful if the original seed URL for each tab could still be accessible, perhaps through a right click on the tab, but the latest viewed instance is the most important. Sync to Android would be a very desirable bonus.

          Reply

        2. John Doe wrote on

          Thank you for fixing Bug 1553982! I have moved most of my work to another browser due to the need to restart so often but will try Firefox again now.

          Reply

        Leave a Reply

        Your email address will not be published. Required fields are marked *

        ================================================ FILE: test/test-pages/folha/expected-metadata.json ================================================ { "title": "Tite diz que errou ao levar taça da Libertadores a Lula em 2012", "byline": "Bruno (Henrique Zecchin) Rodrigues", "dir": null, "lang": "pt-BR", "excerpt": "Na ocasião, técnico do Corinthians entregou réplica do troféu ao ex-presidente", "siteName": "Folha de S.Paulo", "publishedTime": "2018-12-21T12:55:00Z", "readerable": true } ================================================ FILE: test/test-pages/folha/expected.html ================================================

        Após rechaçar um encontro da seleção brasileira com o presidente eleito Jair Bolsonaro, o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012, conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva.

        Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou uma réplica do troféu a Lula.

        "Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV.

        Na ocasião, Tite e outros representantes do Corinthians foram ao Instituto Lula para mostrar a taça original da Libertadores ao ex-presidente.

        O assunto foi levantado porque recentemente Tite foi questionado se aceitaria um encontro da seleção brasileira com Bolsonaro em uma conquista de título ou antes da Copa América de 2019, por exemplo. O treinador deixou claro que preferiria evitar esse tipo de formalidade.

        Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que convidaram Bolsonaro para a festa do título do Campeonato Brasileiro. O presidente eleito até levantou a taça conquistada pelo clube alviverde.

        "Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não", acrescentou o comandante.

        ================================================ FILE: test/test-pages/folha/source.html ================================================ Tite diz que errou ao levar taça da Libertadores a Lula em 2012 - 21/12/2018 - Esporte - Folha
        Assine

        Entrar
        Busca

        Tite diz que errou ao levar taça da Libertadores a Lula em 2012

        Na ocasião, técnico do Corinthians entregou réplica do troféu ao ex-presidente

        Luiz Inácio Lula da Silva (ao centro) recebe de Tite e outros representantes do Corinthians réplica da taça da Libertadores
        Luiz Inácio Lula da Silva (ao centro) recebe de Tite e outros representantes do Corinthians réplica da taça da Libertadores - Heinrich Aikawa/Instituto Lula
        • Ouvir o texto
        São Paulo

        Após rechaçar um encontro da seleção brasileira com o presidente eleito Jair Bolsonaro, o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012, conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva.

        Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou uma réplica do troféu a Lula.

        "Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV.

        Na ocasião, Tite e outros representantes do Corinthians foram ao Instituto Lula para mostrar a taça original da Libertadores ao ex-presidente.

        O assunto foi levantado porque recentemente Tite foi questionado se aceitaria um encontro da seleção brasileira com Bolsonaro em uma conquista de título ou antes da Copa América de 2019, por exemplo. O treinador deixou claro que preferiria evitar esse tipo de formalidade.

        Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que convidaram Bolsonaro para a festa do título do Campeonato Brasileiro. O presidente eleito até levantou a taça conquistada pelo clube alviverde.

        "Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não", acrescentou o comandante.

        UOL
        Tópicos relacionados

        Comentários

        Os comentários não representam a opinião do jornal; a responsabilidade é do autor da mensagem.

        Organização atende 250 crianças e oferece aulas de luta, dança e música

        Saiba como planejar e cumprir suas metas em 2019

        COE pode aumentar potencial de rendimentos sem risco de perder o dinheiro aplicado

        Novas tendências de consumo apontam escolha por praticidade e segurança alimentar

        Banco do Brasil lança app que auxilia produtores rurais

        Aplicação pouco conhecida, COE pode garantir ganhos maiores com capital protegido

        Projetos premiados trabalham a relação tributos e cidadania

        Estímulo à inovação gera interesse estrangeiro no setor aéreo nacional

        Programa TAP Miles&Go otimiza tempo de seus passageiros e agiliza viagem

        Mais lidas em Esporte

        Ver todas
        1. Fenômeno do basquete dos EUA refutou futebol americano e deve chegar à NBA
        2. Militar irá comandar secretaria de esporte do governo Bolsonaro
        3. Globo muda horário, e rodada de quarta no futebol brasileiro será às 21h30
        4. Ana Marcela vê Tóquio-20 como última chance e diz temer intolerância sexual
        5. Quem é o Al Ain, time que desbancou o River Plate no Mundial de Clubes

        Principais do dia

        Covas quebra promessa de Doria e prevê dinheiro de privatização na previdência

        João Doria

        Covas quebra promessa de Doria e prevê dinheiro de privatização na previdência

        Atual gestão tucana em São Paulo havia prometido usar venda de equipamentos na área social

        Pela primeira vez na República, ministério que toma posse excluirá Norte e Nordeste

        Governo Bolsonaro

        Pela primeira vez na República, ministério que toma posse excluirá Norte e Nordeste

        No primeiro escalão, há 11 integrantes do Sudeste, 8 do Sul, 2 do Centro-Oeste e 1 colombiano

        Torçam para que Bolsonaro consiga modernizar Guedes

        Torçam para que Bolsonaro consiga modernizar Guedes

        Ou o presidente vira o conselheiro do ministro da Economia ou a nau afunda

        NEWSLETTER
        ================================================ FILE: test/test-pages/gitlab-blog/expected-metadata.json ================================================ { "title": "3 surprising findings from our 2024 Global DevSecOps Survey", "byline": "Dave Steer", "dir": null, "lang": "en-us", "excerpt": "This year, our survey revealed changes in organizations' investment priorities in the wake of AI — and how AI is shaping the way teams work.", "siteName": "GitLab", "publishedTime": "2024-06-25", "readerable": true } ================================================ FILE: test/test-pages/gitlab-blog/expected.html ================================================

        This year’s survey of more than 5,000 DevSecOps professionals worldwide suggests that as organizations adopt new technologies such as AI, they're reevaluating investment priorities and looking more critically at how they can improve the developer experience. Here's a look at three of the more surprising results from this year's survey, and what they could mean for software development, operations, and security teams in 2024 and beyond.

        1. AI shines a light on cumbersome toolchains

        This year, we looked specifically at how AI might impact DevSecOps teams’ attitudes toward their existing toolchains, and the findings were somewhat surprising. We know AI can help teams simplify software development, but our survey showed that respondents currently using AI might be more frustrated with their toolchains than those not using AI.

        Nearly three-quarters (74%) of respondents whose organizations are currently using AI for software development said they wanted to consolidate their toolchain, compared to 57% of those who aren’t using AI. However, there wasn’t a significant difference between the two groups in the number of tools respondents reported using. In other words, respondents currently using AI weren’t using more tools but still felt a stronger need to consolidate their toolchain.

        Why would AI accelerate the desire to consolidate? One explanation could be that different point solutions running different AI models create unmanageable (and unmeasurable) chaos in the software development lifecycle — and that is shedding new light on organizations’ already cumbersome and counterproductive toolchains. As organizations increase their AI investments, there will be a greater need to improve efficiency by consolidating and simplifying toolchain sprawl. Teams get more value from AI when toolchains are smaller, making integrating AI across the entire software development lifecycle easier.

        One survey respondent identified “too many tools (including AI tools) and context switching” as the biggest challenges in software development in 2024, while another pointed to the “complexity of fragmented landscape of tools across the board.”

        Another respondent highlighted AI’s opportunities to help teams address toolchain challenges: “AI is growing fast, and our current toolchain can be massively improved with AI integrations. We need to train team members better, so they know how to use AI effectively in their daily work.”

        2. AI speeds up developer onboarding — but organizations still have concerns

        Along with the increase in the number of tools teams use, we noted a significant increase in developer onboarding times in this year’s survey. In 2024, 70% of respondents told us it takes developers in their organization more than a month to onboard and become productive, up from 66% in 2023.

        While it’s not surprising that AI-powered chat assistants and code suggestions, can help developers onboard faster, the effect we observed in our survey was dramatic: Respondents who use AI for software development were much more likely to say that developer onboarding typically takes less than a month.

        Despite AI’s clear benefits for developer experience, respondents expressed several concerns about its rapid adoption. Over half (55%) of respondents said introducing AI into the software development lifecycle is risky, and 49% said they fear AI will replace their current role within the next five years.

        Rachel Stephens, senior analyst at industry analyst firm RedMonk, shared her perspective on these findings: “There is a component of psychological safety and team culture that impacts how people feel about AI. Individuals may be concerned about the security or privacy implications of AI, but their sense of unpreparedness may also stem from a feeling that AI has personal risk to their livelihoods.”

        Our take is that the value of AI lies in its ability to automate repetitive tasks and behind-the-scenes optimization, empowering teams to focus on high-level problem-solving, innovation, and value creation. It’s about supplementing — not replacing — the human element of software development. One survey respondent summed this up as follows: “Fostering and maintaining creativity while leaning into AI is a challenge we face. We must remember that AI is simply one tool creative people use to cut out the junk that would otherwise impede productivity. It doesn’t replace human creativity.”

        3. The cloud becomes table stakes

        In our survey, cloud computing has consistently ranked as a top IT investment priority over the past several years. In 2022, cloud computing ranked number two, after security, and in 2023 it took the top spot — not surprising, given increased pressure on organizations to undergo digital transformation.

        In 2024, though, cloud computing saw a sharp decrease, ranking at number five. However, at the same time, it’s clear that the cloud continues to be important. In fact, we saw a significant increase in the number of respondents who said they are running 50% or more of their apps in the cloud. This suggests that while the cloud is still mission-critical for many businesses, it’s now “table stakes” — and at the same time, the list of priorities for technical teams and IT leaders continues to grow.

        According to RedMonk’s Stephens, “We are in a cash-constrained financial environment, and people are having to make prioritization decisions between technology investments — meaning organizations could be reallocating some, but not all, of their digital transformation budgets to things like AI.”

        Explore this year’s report

        Read the full 2024 Global DevSecOps Report for more insights on AI, security, developer experience, and more.

        ================================================ FILE: test/test-pages/gitlab-blog/source.html ================================================ 3 surprising findings from our 2024 Global DevSecOps Survey
        Blog Insights 3 surprising findings from our 2024 Global DevSecOps Survey
        Published on: June 25, 2024
        5 min read

        3 surprising findings from our 2024 Global DevSecOps Survey

        This year, our survey revealed changes in organizations' investment priorities in the wake of AI — and how AI is shaping the way teams work.

        fy25-global-devsecops-report-blog-image
        dsteer-headshot Dave Steer
        developer surveyDevSecOpsAI/MLsecuritynews

        This year’s survey of more than 5,000 DevSecOps professionals worldwide suggests that as organizations adopt new technologies such as AI, they're reevaluating investment priorities and looking more critically at how they can improve the developer experience. Here's a look at three of the more surprising results from this year's survey, and what they could mean for software development, operations, and security teams in 2024 and beyond.

        1. AI shines a light on cumbersome toolchains

        This year, we looked specifically at how AI might impact DevSecOps teams’ attitudes toward their existing toolchains, and the findings were somewhat surprising. We know AI can help teams simplify software development, but our survey showed that respondents currently using AI might be more frustrated with their toolchains than those not using AI.

        Nearly three-quarters (74%) of respondents whose organizations are currently using AI for software development said they wanted to consolidate their toolchain, compared to 57% of those who aren’t using AI. However, there wasn’t a significant difference between the two groups in the number of tools respondents reported using. In other words, respondents currently using AI weren’t using more tools but still felt a stronger need to consolidate their toolchain.

        Why would AI accelerate the desire to consolidate? One explanation could be that different point solutions running different AI models create unmanageable (and unmeasurable) chaos in the software development lifecycle — and that is shedding new light on organizations’ already cumbersome and counterproductive toolchains. As organizations increase their AI investments, there will be a greater need to improve efficiency by consolidating and simplifying toolchain sprawl. Teams get more value from AI when toolchains are smaller, making integrating AI across the entire software development lifecycle easier.

        One survey respondent identified “too many tools (including AI tools) and context switching” as the biggest challenges in software development in 2024, while another pointed to the “complexity of fragmented landscape of tools across the board.”

        Another respondent highlighted AI’s opportunities to help teams address toolchain challenges: “AI is growing fast, and our current toolchain can be massively improved with AI integrations. We need to train team members better, so they know how to use AI effectively in their daily work.”

        2. AI speeds up developer onboarding — but organizations still have concerns

        Along with the increase in the number of tools teams use, we noted a significant increase in developer onboarding times in this year’s survey. In 2024, 70% of respondents told us it takes developers in their organization more than a month to onboard and become productive, up from 66% in 2023.

        While it’s not surprising that AI-powered chat assistants and code suggestions, can help developers onboard faster, the effect we observed in our survey was dramatic: Respondents who use AI for software development were much more likely to say that developer onboarding typically takes less than a month.

        Despite AI’s clear benefits for developer experience, respondents expressed several concerns about its rapid adoption. Over half (55%) of respondents said introducing AI into the software development lifecycle is risky, and 49% said they fear AI will replace their current role within the next five years.

        Rachel Stephens, senior analyst at industry analyst firm RedMonk, shared her perspective on these findings: “There is a component of psychological safety and team culture that impacts how people feel about AI. Individuals may be concerned about the security or privacy implications of AI, but their sense of unpreparedness may also stem from a feeling that AI has personal risk to their livelihoods.”

        Our take is that the value of AI lies in its ability to automate repetitive tasks and behind-the-scenes optimization, empowering teams to focus on high-level problem-solving, innovation, and value creation. It’s about supplementing — not replacing — the human element of software development. One survey respondent summed this up as follows: “Fostering and maintaining creativity while leaning into AI is a challenge we face. We must remember that AI is simply one tool creative people use to cut out the junk that would otherwise impede productivity. It doesn’t replace human creativity.”

        3. The cloud becomes table stakes

        In our survey, cloud computing has consistently ranked as a top IT investment priority over the past several years. In 2022, cloud computing ranked number two, after security, and in 2023 it took the top spot — not surprising, given increased pressure on organizations to undergo digital transformation.

        In 2024, though, cloud computing saw a sharp decrease, ranking at number five. However, at the same time, it’s clear that the cloud continues to be important. In fact, we saw a significant increase in the number of respondents who said they are running 50% or more of their apps in the cloud. This suggests that while the cloud is still mission-critical for many businesses, it’s now “table stakes” — and at the same time, the list of priorities for technical teams and IT leaders continues to grow.

        According to RedMonk’s Stephens, “We are in a cash-constrained financial environment, and people are having to make prioritization decisions between technology investments — meaning organizations could be reallocating some, but not all, of their digital transformation budgets to things like AI.”

        Explore this year’s report

        Read the full 2024 Global DevSecOps Report for more insights on AI, security, developer experience, and more.

        More to explore

        View all blog posts
        Insights
        navigation.jpeg

        How we overhauled GitLab navigation

        Insights
        beautify.jpg

        Beautifying our UI: Giving GitLab build features a fresh look

        Insights
        2023-devsecops-report-blog-banner2.png

        4 best practices leading orgs to release software faster

        We want to hear from you

        Enjoyed reading this blog post or have questions or feedback? Share your thoughts by creating a new topic in the GitLab community forum. Share your feedback

        Ready to get started?

        See what your team could do with a unified DevSecOps Platform.

        Get free trial

        New to GitLab and not sure where to start?

        Get started guide

        Learn about what GitLab can do for your team

        Talk to an expert
        ================================================ FILE: test/test-pages/gmw/expected-metadata.json ================================================ { "title": "宇航员在太空中喝酒会怎么样？后果很严重 _探索者 _光明网", "byline": "肖春芳", "dir": null, "excerpt": "不幸的是，对于希望能喝上一杯的太空探险者，那些将他们送上太空的政府机构普遍禁止他们染指包括酒在内的含酒精饮料。", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/gmw/expected.html ================================================

        　　翱翔于距地球数千公里的太空中，进入广袤漆黑的未知领域，是一项艰苦卓绝的工作。这让人感到巨大压力和极度恐慌。那么，为什么不能让宇航员来一杯“地球末日”鸡尾酒来放松一下？

        　　不幸的是，对于希望能喝上一杯的太空探险者，那些将他们送上太空的政府机构普遍禁止他们染指包括酒在内的含酒精饮料。

        　　但是，很快普通人都会有机会向人类“最终的边疆”出发——以平民化旅行的形式，去探索和殖民火星。确实，火星之旅将是一次令人感到痛苦的旅行，可能一去不复返并要几年时间才能完成，但是否应该允许参与者在旅程中痛饮一番？或至少携带能在火星上发酵自制酒精饮料的设备？

        (Credit: Nasa)

        　　图注：巴兹?奥尔德林(Buzz Aldrin)可能是第二个在月球上行走的人，但他是第一个在月球上喝酒的人

        　　事实是，历史上酒与太空探险有一种复杂的关系。让我们来看看喝了酒的航天员究竟会发生什么—— 如果我们开始给予进入太空的人类更大的自由度，又可能会发生什么。

        　　人们普遍认为，当一个人所处的海拔越高，喝醉后会越容易感到头昏。因此，人们自然地想到，当人身处地球轨道上时，饮酒会对人体有更强烈的致眩作用。但这种说法可能不是正确的。

        　　事实上，有证据表明，早在上世纪八十年代就澄清了这一传言。1985年，美国联邦航空管理局（UFAA）开展了一项研究，以验证人在不同的海拔高度饮酒，是否会影响执行复杂任务时的表现和酒精测定仪的读数。

        　　在这项研究中，17名男子被要求在地面和一间模拟海拔3.7公里的房间内喝下一些伏特加。然后，他们被要求完成各种任务，包括心算口算问题、用操纵杆在示波器上跟踪灯光以及各种其它测试。研究人员发现，“酒精和海拔高度对酒精测定仪读数或完成任务的表现情况没有交互作用”。

        　　所以，人乘坐飞机时醉得更快是个传说？纽约州立大学（State University of New York，SUNY）社会学荣誉教授戴夫·汉森（Dave Hanson）研究酒精和饮酒超过40年，他认为确实如此。他说：“我不认为它（在太空中饮酒）会有任何不同。”

        　　他认为高原反应可能类似于宿醉，但它也可能类似于中毒。他说：“如果人们没有感受到充分的大气压力，他们也会觉得喝醉了一样。”

        　　相反，那些声称在飞机上比在地面上醉得更快的人，可能只是经历了“自认喝醉（think-drink）”效应，这种效应多年来已被广泛研究。它表明，如果人们认为自己喝醉了，那他们的一举一动会真的像喝醉了一样—— 而不是实际上他们真的醉了。

        　　汉森指出：“如果人们脑子里一直认为在飞机上酒精会对他们产生与平常不同的作用，那么他们乘坐飞机时真的会觉得酒精对他们产生了不同的作用。”

        　　所以，如果酒精对人体的物理效应与海拔高度无关，那么在国际空间站上睡前小饮一杯不应该是一个大问题，对吧？错了。

        　　美国宇航局约翰逊航天中心发言人丹尼尔·霍特(Daniel Huot)表示：“国际空间站上的宇航员不允许喝酒。在国际空间站上，酒精和其它挥发性化合物的使用受到控制，因为它们的挥发物可能对该站的水回收系统产生影响。”

        　　为此，国际空间站上的宇航员甚至没有被提供含有酒精的产品，例如漱口水、香水或须后水。如果在国际空间站上饮酒狂欢，溢出的啤酒也可能存在损坏设备的风险。

        (Credit: iStock)

        　　图注：测试表明，有关人在高空中喝酒更容易醉的传言是不正确的

        　　然后是责任的问题。我们不允许汽车司机或飞机飞行员喝醉后驾驶，所以并不奇怪同样的规则适用于国际空间站上的宇航员。毕竟国际空间站的造价高达1500亿美元，而且在接近真空的太空中其运行速度达到了每小时27680公里。

        　　然而，2007年，美国宇航局(NASA)成立了一个负责调查宇航员健康状况的独立小组，称历史上该机构至少有两名宇航员在即将飞行前喝了大量的酒，但仍然被允许飞行。Nasa安全负责人随后的审查发现并没有证据支持这一指控。宇航员在飞行前12小时是严禁饮酒的，因为他们需要充分的思维能力和清醒的意识。

        　　出台这一规则的原因很清楚。在1985年UFAA开展的关于酒精在不同海拔高度影响的研究中，研究人员得出结论，酒精的影响与海拔高度无关。无论参与测试的人员在什么海拔高度喝酒，其酒精测量仪的读数都是一样的。他们的行为表现受到的影响也相同，但如果提供给测试人员的是安慰剂，则身处高空比身处海平面的行为表现要更差一些。这表明，无论是否摄入酒精，海拔高度可能对心理表现有轻微的影响。

        　　国际空间站禁止享用啤酒等有大量泡沫的饮料，可能有另一个原因：没有重力的帮助，液体和气体会在宇航员的胃里不停地翻滚，导致他们不断地打嗝。

        　　然而，尽管有严格的规则，这并不意味着太空中的人类不会接触发酵液体。在国际空间站上进行了大量有关酒精的实验—— 但没有发生让众人去饮酒的情况，所以没有人真正了解太空中人体对酒精具体有怎样的反应。

        　　NASA发言人斯蒂芬妮?席尔霍尔茨(Stephanie Schierhol）表示：“我们研究了太空中宇航员身体的各种变化，包括微生物层面的。我们有一个营养计划，以确保他们的身体获得保持健康所需要的营养。显然，在实施‘天空实验室(skylab)’项目时，他们曾将雪利酒与宇航员一起送到太空中，但宇航员在零重力飞行时使用雪利酒的测试结果不太好。”天空实验室是美国第一座空间站。

        　　席尔霍尔茨补充说，在测试中使用雪利酒“引发呕吐反射，公众也反对”。

        　　也许最令人惊讶的是，人类在月球表面上喝的第一种液体是葡萄酒。前NASA宇航员巴兹·奥尔德林(Buzz Aldrin)在采访和他撰写的书中表示，1969年，在和尼尔·阿姆斯特朗(Neil Armstrong)走出登月舱之前的圣餐仪式上，他喝了少量葡萄酒。举行这一仪式时与地面的通信出现了暂停，因此这一过程从来没有播出。

        　　虽然Nasa对太空中酒精的使用有严格的规定，但在这方面俄罗斯过去似乎更为宽松。在其“和平号”空间站上，宇航员允许喝点干邑和伏特加。当他们发现国际空间站将严格禁止饮酒时，显然有不少怨言。

        　　然而，奇怪的是，酒仍然能通过各种方式出现在国际空间站上。2015年，日本酿酒商三得利(Suntory)的全球创新中心将该公司一些获奖的威士忌运送到国际空间站，参与一项旨在验证“能否通过利用微重力环境增强酒精饮料醇厚性”的实验。换句话说，在微重力下酒的陈酿过程可能不同，导致它的陈酿进程更快、味道更好。对此，地球上的每家酿酒商都想进一步地了解。

        　　几年前，即2011年9月至2014年9月，Nasa赞助了一个试验，研究微重力环境对威士忌中未发酵麦芽与烧焦橡木颗粒的影响，这两种物质能对威士忌的陈酿起帮助作用。在太空中逗留将近1000天后，用于测试的威士忌的单宁成分保持不变——但是太空中橡木颗粒产生了更高浓度的木质素分解产物，这种物质能赋予威士忌特别的风味。

        　　Nasa表示：“这种试验不仅对麦芽威士忌行业有影响，而且对整个食品和饮料行业也有影响。送上太空的威士忌与对照样品之间的风味差异是如此显著，需要进一步分析以破解不同口味产生的原因。”

        　　因此，即使宇航员自己被禁止在地球轨道上饮酒，但他们正在做的工作可以提高在地上消费的酒的质量。

        　　相比之下，执行登陆火星任务的人将远离家乡几年，而不是几个月，因此可能会有人提出有关禁止饮酒的规定可以放松一些。

        　　然而，像戴夫?汉森这样的专家认为，继续禁止饮酒并没有什么害处。除了实际的安全问题，饮酒还可能有其它挑战。汉森认为，地球人存在许多社会文化方面的差异，而且人连续几年时间呆在一个狭小的空间里，很容易突然发怒，这些因素都使饮酒问题变得很棘手。

        (Credit: David Frohman/Peachstate Historical Consulting Inc)

        　　图注：奥尔德林的圣餐杯回到了地球上

        　　他说：“这是一个政治问题，也是一个文化方面的问题，但不是一个科学上的问题。这将是未来一个可能产生冲突领域，因为人们具有不同的文化背景，他们对饮酒的态度不同。”他进一步指出，如果你与穆斯林、摩门教徒或禁酒主义者分配在同一间宿舍怎么办？面对未来人们可能在一个没有期限的时间内呆在一个有限的空间里，需要“尽早解决”如何协调不同文化观点的问题。

        　　所以，当宇航员在地球轨道上时，将还不得不满足于通过欣赏外面的景色来振作精神，而不要指望沉溺于烈酒中。我们留在地球上的人，则可以准备好适量的香槟酒，以迎接他们的归来。

        　　原标题：他晚于阿姆斯特朗登月 却是首个敢在月球喝酒的人

        　　出品︱网易科学人栏目组 胖胖

        　　作者︱春春

        [责任编辑:肖春芳]

        ================================================ FILE: test/test-pages/gmw/source.html ================================================ 宇航员在太空中喝酒会怎么样？后果很严重 _探索者 _光明网
      • 时政
      • 国际
      • 时评
      • 理论
      • 文化
      • 科技
      • 教育
      • 经济
      • 生活
      • 法治
      • 军事
      • 卫生
      • 养生
      • 女人
      • 娱乐
      • 电视
      • 图片
      • 博客
      • 棋牌
      • 光明报系
      • 更多>>

        • 宇航员在太空中喝酒会怎么样？后果很严重

        2017-03-10 09:58　来源：网易科学人　
        2017-03-10 09:58:03来源：网易科学人作者：责任编辑：肖春芳

        　　翱翔于距地球数千公里的太空中，进入广袤漆黑的未知领域，是一项艰苦卓绝的工作。这让人感到巨大压力和极度恐慌。那么，为什么不能让宇航员来一杯“地球末日”鸡尾酒来放松一下？

        　　不幸的是，对于希望能喝上一杯的太空探险者，那些将他们送上太空的政府机构普遍禁止他们染指包括酒在内的含酒精饮料。

        　　但是，很快普通人都会有机会向人类“最终的边疆”出发——以平民化旅行的形式，去探索和殖民火星。确实，火星之旅将是一次令人感到痛苦的旅行，可能一去不复返并要几年时间才能完成，但是否应该允许参与者在旅程中痛饮一番？或至少携带能在火星上发酵自制酒精饮料的设备？

        (Credit: Nasa)

        　　图注：巴兹?奥尔德林(Buzz Aldrin)可能是第二个在月球上行走的人，但他是第一个在月球上喝酒的人

        　　事实是，历史上酒与太空探险有一种复杂的关系。让我们来看看喝了酒的航天员究竟会发生什么—— 如果我们开始给予进入太空的人类更大的自由度，又可能会发生什么。

        　　人们普遍认为，当一个人所处的海拔越高，喝醉后会越容易感到头昏。因此，人们自然地想到，当人身处地球轨道上时，饮酒会对人体有更强烈的致眩作用。但这种说法可能不是正确的。

        　　事实上，有证据表明，早在上世纪八十年代就澄清了这一传言。1985年，美国联邦航空管理局（UFAA）开展了一项研究，以验证人在不同的海拔高度饮酒，是否会影响执行复杂任务时的表现和酒精测定仪的读数。

        　　在这项研究中，17名男子被要求在地面和一间模拟海拔3.7公里的房间内喝下一些伏特加。然后，他们被要求完成各种任务，包括心算口算问题、用操纵杆在示波器上跟踪灯光以及各种其它测试。研究人员发现，“酒精和海拔高度对酒精测定仪读数或完成任务的表现情况没有交互作用”。

        　　所以，人乘坐飞机时醉得更快是个传说？纽约州立大学（State University of New York，SUNY）社会学荣誉教授戴夫·汉森（Dave Hanson）研究酒精和饮酒超过40年，他认为确实如此。他说：“我不认为它（在太空中饮酒）会有任何不同。”

        　　他认为高原反应可能类似于宿醉，但它也可能类似于中毒。他说：“如果人们没有感受到充分的大气压力，他们也会觉得喝醉了一样。”

        　　相反，那些声称在飞机上比在地面上醉得更快的人，可能只是经历了“自认喝醉（think-drink）”效应，这种效应多年来已被广泛研究。它表明，如果人们认为自己喝醉了，那他们的一举一动会真的像喝醉了一样—— 而不是实际上他们真的醉了。

        　　汉森指出：“如果人们脑子里一直认为在飞机上酒精会对他们产生与平常不同的作用，那么他们乘坐飞机时真的会觉得酒精对他们产生了不同的作用。”

        　　所以，如果酒精对人体的物理效应与海拔高度无关，那么在国际空间站上睡前小饮一杯不应该是一个大问题，对吧？错了。

        　　美国宇航局约翰逊航天中心发言人丹尼尔·霍特(Daniel Huot)表示：“国际空间站上的宇航员不允许喝酒。在国际空间站上，酒精和其它挥发性化合物的使用受到控制，因为它们的挥发物可能对该站的水回收系统产生影响。”

        　　为此，国际空间站上的宇航员甚至没有被提供含有酒精的产品，例如漱口水、香水或须后水。如果在国际空间站上饮酒狂欢，溢出的啤酒也可能存在损坏设备的风险。

        (Credit: iStock)

        　　图注：测试表明，有关人在高空中喝酒更容易醉的传言是不正确的

        　　然后是责任的问题。我们不允许汽车司机或飞机飞行员喝醉后驾驶，所以并不奇怪同样的规则适用于国际空间站上的宇航员。毕竟国际空间站的造价高达1500亿美元，而且在接近真空的太空中其运行速度达到了每小时27680公里。

        　　然而，2007年，美国宇航局(NASA)成立了一个负责调查宇航员健康状况的独立小组，称历史上该机构至少有两名宇航员在即将飞行前喝了大量的酒，但仍然被允许飞行。Nasa安全负责人随后的审查发现并没有证据支持这一指控。宇航员在飞行前12小时是严禁饮酒的，因为他们需要充分的思维能力和清醒的意识。

        　　出台这一规则的原因很清楚。在1985年UFAA开展的关于酒精在不同海拔高度影响的研究中，研究人员得出结论，酒精的影响与海拔高度无关。无论参与测试的人员在什么海拔高度喝酒，其酒精测量仪的读数都是一样的。他们的行为表现受到的影响也相同，但如果提供给测试人员的是安慰剂，则身处高空比身处海平面的行为表现要更差一些。这表明，无论是否摄入酒精，海拔高度可能对心理表现有轻微的影响。

        　　国际空间站禁止享用啤酒等有大量泡沫的饮料，可能有另一个原因：没有重力的帮助，液体和气体会在宇航员的胃里不停地翻滚，导致他们不断地打嗝。

        　　然而，尽管有严格的规则，这并不意味着太空中的人类不会接触发酵液体。在国际空间站上进行了大量有关酒精的实验—— 但没有发生让众人去饮酒的情况，所以没有人真正了解太空中人体对酒精具体有怎样的反应。

        　　NASA发言人斯蒂芬妮?席尔霍尔茨(Stephanie Schierhol）表示：“我们研究了太空中宇航员身体的各种变化，包括微生物层面的。我们有一个营养计划，以确保他们的身体获得保持健康所需要的营养。显然，在实施‘天空实验室(skylab)’项目时，他们曾将雪利酒与宇航员一起送到太空中，但宇航员在零重力飞行时使用雪利酒的测试结果不太好。”天空实验室是美国第一座空间站。

        　　席尔霍尔茨补充说，在测试中使用雪利酒“引发呕吐反射，公众也反对”。

        　　也许最令人惊讶的是，人类在月球表面上喝的第一种液体是葡萄酒。前NASA宇航员巴兹·奥尔德林(Buzz Aldrin)在采访和他撰写的书中表示，1969年，在和尼尔·阿姆斯特朗(Neil Armstrong)走出登月舱之前的圣餐仪式上，他喝了少量葡萄酒。举行这一仪式时与地面的通信出现了暂停，因此这一过程从来没有播出。

        　　虽然Nasa对太空中酒精的使用有严格的规定，但在这方面俄罗斯过去似乎更为宽松。在其“和平号”空间站上，宇航员允许喝点干邑和伏特加。当他们发现国际空间站将严格禁止饮酒时，显然有不少怨言。

        　　然而，奇怪的是，酒仍然能通过各种方式出现在国际空间站上。2015年，日本酿酒商三得利(Suntory)的全球创新中心将该公司一些获奖的威士忌运送到国际空间站，参与一项旨在验证“能否通过利用微重力环境增强酒精饮料醇厚性”的实验。换句话说，在微重力下酒的陈酿过程可能不同，导致它的陈酿进程更快、味道更好。对此，地球上的每家酿酒商都想进一步地了解。

        　　几年前，即2011年9月至2014年9月，Nasa赞助了一个试验，研究微重力环境对威士忌中未发酵麦芽与烧焦橡木颗粒的影响，这两种物质能对威士忌的陈酿起帮助作用。在太空中逗留将近1000天后，用于测试的威士忌的单宁成分保持不变——但是太空中橡木颗粒产生了更高浓度的木质素分解产物，这种物质能赋予威士忌特别的风味。

        　　Nasa表示：“这种试验不仅对麦芽威士忌行业有影响，而且对整个食品和饮料行业也有影响。送上太空的威士忌与对照样品之间的风味差异是如此显著，需要进一步分析以破解不同口味产生的原因。”

        　　因此，即使宇航员自己被禁止在地球轨道上饮酒，但他们正在做的工作可以提高在地上消费的酒的质量。

        　　相比之下，执行登陆火星任务的人将远离家乡几年，而不是几个月，因此可能会有人提出有关禁止饮酒的规定可以放松一些。

        　　然而，像戴夫?汉森这样的专家认为，继续禁止饮酒并没有什么害处。除了实际的安全问题，饮酒还可能有其它挑战。汉森认为，地球人存在许多社会文化方面的差异，而且人连续几年时间呆在一个狭小的空间里，很容易突然发怒，这些因素都使饮酒问题变得很棘手。

        (Credit: David Frohman/Peachstate Historical Consulting Inc)

        　　图注：奥尔德林的圣餐杯回到了地球上

        　　他说：“这是一个政治问题，也是一个文化方面的问题，但不是一个科学上的问题。这将是未来一个可能产生冲突领域，因为人们具有不同的文化背景，他们对饮酒的态度不同。”他进一步指出，如果你与穆斯林、摩门教徒或禁酒主义者分配在同一间宿舍怎么办？面对未来人们可能在一个没有期限的时间内呆在一个有限的空间里，需要“尽早解决”如何协调不同文化观点的问题。

        　　所以，当宇航员在地球轨道上时，将还不得不满足于通过欣赏外面的景色来振作精神，而不要指望沉溺于烈酒中。我们留在地球上的人，则可以准备好适量的香槟酒，以迎接他们的归来。

        　　原标题：他晚于阿姆斯特朗登月 却是首个敢在月球喝酒的人

        　　出品︱网易科学人栏目组 胖胖

        　　作者︱春春

        [责任编辑:肖春芳]
        分享
        0
        您此时的心情
        新闻表情排行 日/周

        [值班总编推荐] 想想童乞的遭遇，谁都会坐立不安

        [值班总编推荐] 网络慈善：珍视每一份爱心

        [值班总编推荐] 被村上春树打脸，日本右翼急了


        老人制万根“爱心拐棍”

        公安破获象牙制售案件

        古稀老人照顾病妻26年

        维和部队营救演习侧记
        图说能源

        工业遗迹的二次生命

        电力作业上的“瓦力”

        芳烃技术有多牛？

        EFG2016新看点

        手机光明网

        光明网版权所有

        光明日报社概况 | 关于光明网 | 报网动态 | 联系我们 | 法律声明 | 光明员工 | 光明网邮箱 | 网站地图

        光明网版权所有

        立即打开
        X
        选择其他平台 >>
        分享到
        ================================================ FILE: test/test-pages/google-sre-book-1/expected-metadata.json ================================================ { "title": "Google - Site Reliability Engineering", "byline": "Written by Rob Ewaschuk\n Edited by Betsy Beyer", "dir": null, "lang": "en", "excerpt": "Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/google-sre-book-1/expected.html ================================================

        Monitoring Distributed Systems

        Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.

        Definitions

        There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.

        Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.

        Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.

        Testing externally visible behavior as a user would see it.

        An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.

        A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as tickets, email alerts,22 and pages.

        A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, and insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.

        Node and machine

        Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple services worth monitoring on a single machine. The services may either be:

        • Related to each other: for example, a caching server and a web server
        • Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like Puppet or Chef

        Any change to a service’s running software or its configuration.

        Why Monitor?

        There are many reasons to monitor a system, including:

        How big is my database and how fast is it growing? How quickly is my daily-active user count growing?

        Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?

        Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.

        Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in The Four Golden Signals).

        Our latency just shot up; what else happened around the same time?

        System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.

        Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."

        Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.

        Setting Reasonable Expectations for Monitoring

        Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)

        In general, Google has trended toward simpler and faster monitoring systems, with better tools for post hoc analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.

        Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.

        Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.

        Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.

        Symptoms Versus Causes

        Your monitoring system should address two questions: what’s broken, and why?

        The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. Table 6-1 lists some hypothetical symptoms and corresponding causes.

        Table 6-1. Example symptoms and causes
        Symptom Cause

        I’m serving HTTP 500s or 404s

        Database servers are refusing connections

        My responses are slow

        CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss

        Users in Antarctica aren’t receiving animated cat GIFs

        Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs

        Private content is world-readable

        A new software push caused ACLs to be forgotten and allowed all requests

        "What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.

        Black-Box Versus White-Box

        We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.

        Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.

        When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.

        For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.

        The Four Golden Signals

        The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.

        The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.

        A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per second.

        The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.

        How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.

        In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.

        Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."

        If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.

        Worrying About Your Tail (or, Instrumentation and Performance)

        When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.23 If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your frontend.

        The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.

        Choosing an Appropriate Resolution for Measurements

        Different aspects of a system should be measured with different levels of granularity. For example:

        • Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
        • On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
        • Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.

        Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:

        1. Record the current CPU utilization each second.
        2. Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
        3. Aggregate those values every minute.

        This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.

        As Simple as Possible, No Simpler

        Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:

        • Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
        • Extra code to detect and expose possible causes
        • Associated dashboards for each of these possible causes

        The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.

        Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:

        • The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
        • Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
        • Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.

        In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).

        Tying These Principles Together

        The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.

        When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:24

        • Does this rule detect an otherwise undetected condition that is urgent, actionable, and actively or imminently user-visible?25
        • Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
        • Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
        • Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
        • Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?

        These questions reflect a fundamental philosophy on pages and pagers:

        • Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
        • Every page should be actionable.
        • Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
        • Pages should be about a novel problem or an event that hasn’t been seen before.

        Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.

        Monitoring for the Long Term

        In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.

        It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.

        Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see Service Level Objectives). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.

        Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.

        To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.

        This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.

        In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.

        At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.

        To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.

        This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.

        Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.

        A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall level of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.

        Conclusion

        A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.

        Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.

        22Sometimes known as "alert spam," as they are rarely read or acted on.

        23If 1% of your requests are 50x the average, it means that the rest of your requests are about twice as fast as the average. But if you’re not measuring your distribution, the idea that most of your requests are near the mean is just hopeful thinking.

        24See Applying Cardiac Alarm Management Techniques to Your On-Call [Hol14] for an example of alert fatigue in another context.

        25Zero-redundancy (N + 0) situations count as imminent, as do "nearly full" parts of your service! For more details about the concept of redundancy, see https://en.wikipedia.org/wiki/N%2B1_redundancy.

        ================================================ FILE: test/test-pages/google-sre-book-1/source.html ================================================ Google - Site Reliability Engineering

        Chapter 6 - Monitoring Distributed Systems

        1. Table of Contents
        2. Foreword
        3. Preface
        4. Part I - Introduction
        5. 1. Introduction
        6. 2. The Production Environment at Google, from the Viewpoint of an SRE
        7. Part II - Principles
        8. 3. Embracing Risk
        9. 4. Service Level Objectives
        10. 5. Eliminating Toil
        11. 6. Monitoring Distributed Systems
        12. 7. The Evolution of Automation at Google
        13. 8. Release Engineering
        14. 9. Simplicity
        15. Part III - Practices
        16. 10. Practical Alerting
        17. 11. Being On-Call
        18. 12. Effective Troubleshooting
        19. 13. Emergency Response
        20. 14. Managing Incidents
        21. 15. Postmortem Culture: Learning from Failure
        22. 16. Tracking Outages
        23. 17. Testing for Reliability
        24. 18. Software Engineering in SRE
        25. 19. Load Balancing at the Frontend
        26. 20. Load Balancing in the Datacenter
        27. 21. Handling Overload
        28. 22. Addressing Cascading Failures
        29. 23. Managing Critical State: Distributed Consensus for Reliability
        30. 24. Distributed Periodic Scheduling with Cron
        31. 25. Data Processing Pipelines
        32. 26. Data Integrity: What You Read Is What You Wrote
        33. 27. Reliable Product Launches at Scale
        34. Part IV - Management
        35. 28. Accelerating SREs to On-Call and Beyond
        36. 29. Dealing with Interrupts
        37. 30. Embedding an SRE to Recover from Operational Overload
        38. 31. Communication and Collaboration in SRE
        39. 32. The Evolving SRE Engagement Model
        40. Part V - Conclusions
        41. 33. Lessons Learned from Other Industries
        42. 34. Conclusion
        43. Appendix A. Availability Table
        44. Appendix B. A Collection of Best Practices for Production Services
        45. Appendix C. Example Incident State Document
        46. Appendix D. Example Postmortem
        47. Appendix E. Launch Coordination Checklist
        48. Appendix F. Example Production Meeting Minutes
        49. Bibliography

        Monitoring Distributed Systems

        Written by Rob Ewaschuk
        Edited by Betsy Beyer

        Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.

        Definitions

        There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.

        Monitoring

        Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.

        White-box monitoring

        Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.

        Black-box monitoring

        Testing externally visible behavior as a user would see it.

        Dashboard

        An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.

        Alert

        A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as tickets, email alerts,22 and pages.

        Root cause

        A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, and insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.

        Node and machine

        Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple services worth monitoring on a single machine. The services may either be:

        • Related to each other: for example, a caching server and a web server
        • Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like Puppet or Chef
        Push

        Any change to a service’s running software or its configuration.

        Why Monitor?

        There are many reasons to monitor a system, including:

        How big is my database and how fast is it growing? How quickly is my daily-active user count growing?

        Comparing over time or experiment groups

        Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?

        Alerting

        Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.

        Building dashboards

        Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in The Four Golden Signals).

        Conducting ad hoc retrospective analysis (i.e., debugging)

        Our latency just shot up; what else happened around the same time?

        System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.

        Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."

        Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.

        Setting Reasonable Expectations for Monitoring

        Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)

        In general, Google has trended toward simpler and faster monitoring systems, with better tools for post hoc analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.

        Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.

        Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.

        Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.

        Symptoms Versus Causes

        Your monitoring system should address two questions: what’s broken, and why?

        The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. Table 6-1 lists some hypothetical symptoms and corresponding causes.

        Table 6-1. Example symptoms and causes
        Symptom Cause

        I’m serving HTTP 500s or 404s

        Database servers are refusing connections

        My responses are slow

        CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss

        Users in Antarctica aren’t receiving animated cat GIFs

        Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs

        Private content is world-readable

        A new software push caused ACLs to be forgotten and allowed all requests

        "What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.

        Black-Box Versus White-Box

        We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.

        Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.

        When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.

        For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.

        The Four Golden Signals

        The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.

        Latency

        The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.

        Traffic

        A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per second.

        Errors

        The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.

        Saturation

        How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.

        In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.

        Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."

        If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.

        Worrying About Your Tail (or, Instrumentation and Performance)

        When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.23 If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your frontend.

        The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.

        Choosing an Appropriate Resolution for Measurements

        Different aspects of a system should be measured with different levels of granularity. For example:

        • Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
        • On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
        • Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.

        Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:

        1. Record the current CPU utilization each second.
        2. Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
        3. Aggregate those values every minute.

        This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.

        As Simple as Possible, No Simpler

        Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:

        • Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
        • Extra code to detect and expose possible causes
        • Associated dashboards for each of these possible causes

        The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.

        Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:

        • The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
        • Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
        • Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.

        In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).

        Tying These Principles Together

        The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.

        When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:24

        • Does this rule detect an otherwise undetected condition that is urgent, actionable, and actively or imminently user-visible?25
        • Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
        • Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
        • Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
        • Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?

        These questions reflect a fundamental philosophy on pages and pagers:

        • Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
        • Every page should be actionable.
        • Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
        • Pages should be about a novel problem or an event that hasn’t been seen before.

        Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.

        Monitoring for the Long Term

        In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.

        It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.

        Bigtable SRE: A Tale of Over-Alerting

        Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see Service Level Objectives). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.

        Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.

        To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.

        This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.

        Gmail: Predictable, Scriptable Responses from Humans

        In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.

        At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.

        To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.

        This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.

        Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.

        The Long Run

        A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall level of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.

        Conclusion

        A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.

        Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.

        22Sometimes known as "alert spam," as they are rarely read or acted on.

        23If 1% of your requests are 50x the average, it means that the rest of your requests are about twice as fast as the average. But if you’re not measuring your distribution, the idea that most of your requests are near the mean is just hopeful thinking.

        24See Applying Cardiac Alarm Management Techniques to Your On-Call [Hol14] for an example of alert fatigue in another context.

        25Zero-redundancy (N + 0) situations count as imminent, as do "nearly full" parts of your service! For more details about the concept of redundancy, see https://en.wikipedia.org/wiki/N%2B1_redundancy.

        Copyright © 2017 Google, Inc. Published by O'Reilly Media, Inc. Licensed under CC BY-NC-ND 4.0

        ================================================ FILE: test/test-pages/guardian-1/expected-metadata.json ================================================ { "title": "'What is the sea telling us?': Māori tribes fearful over whale strandings | Eleanor Ainge Roy", "byline": "Eleanor Ainge Roy", "dir": null, "lang": "en", "excerpt": "New Zealand’s whale whisperers worry that manmade changes in the ocean are behind the spike in beachings", "siteName": "the Guardian", "publishedTime": "2019-01-03T07:00:02.000Z", "readerable": true } ================================================ FILE: test/test-pages/guardian-1/expected.html ================================================

        Whale whisperer Hori Parata was just seven years old when he attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland, their cries screeching through the air on the deserted stretch of sand.

        Seven decades later, Parata, 75, has now overseen more than 500 strandings and is renowned in New Zealand as the leading Māori whale expert, called on by tribes around the country for cultural guidance as marine strandings become increasingly complex and fatal.

        “Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and uncompromising elder of the Ngātiwai tribe of eastern Northland.

        Hori Parata at his Pātaua farm, the place where he was born and grew up.

        • Hori Parata at his Pātaua farm, the place where he was born and grew up

        “We’re having to put up with a lot of stuff today. The public want to hug the whales, they want to touch them, they want to feel good – that’s not the thing. We feel that is ridiculous.”

        Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale stranding capital of the world, with more than 5,000 incidents recorded since 1840, and an average of 300 individual animals beaching themselves each year.

        Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Maōri design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata

        • Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a traditional Māori design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father

        Concrete information on why whales strand remains elusive, but “sickness, navigational error, geographical features, a rapidly falling tide, being chased by a predator, or extreme weather” are all thought to contribute, according to the New Zealand Department of Conservation.

        Climate change is to blame too, scientists think, with warming ocean temperatures moving whales’ prey closer to the shore and forcing them to pursue their food into shallow waters.

        A bin of small whale bones.
        The baleen recovered from a stranded Pygmy Right Whale.
        Squid beaks, from the stomach of a Sperm Whale.

        • Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm whale; the baleen filter-feeder system recovered from a stranded pygmy right whale.

        ‘Unprecedented’ strandings

        November marked the beginning of whale stranding season, and it started with a surge in incidents, according to whale rescue group Project Jonah, with 140 pilot whales beaching and dying on Stewart Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham Islands and a spate of individual cases around the country.

        And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on their flesh – an acute sense of grief is growing among New Zealand’s indigenous people, who regard whales as their ancestors and taonga (treasures).

        “These days it is like a zoo. People just want to come and gawk at us, without even trying to understand what is happening with the animals and the environment,” says Parata, bristling with anger.

        whale strandings

        “When will we talk about what is hurting these animals out on the sea? They are drowning out there, they can’t breathe, they beach themselves to be with the Aunties.”

        Ngātiwai believe the whales beach when they are ready to die and want to return to their families, the Māori people. Then, their human families use the whales’ gift of their bodies for sacred carvings, for traditional medicines, and even for compost.

        There are marked tribal differences across New Zealand and while some tribes work to refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the Department of Conservation and volunteer groups to take the lead in rescue efforts.

        Then the tribe moves in en masse and holds a karakia (prayer), names each animal and sets to work removing their bones, blubber, eyes and teeth for cultural purposes.

        Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata.

        • Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is storing a pair of massive sperm whale jawbones. Cullen is an integral member of Hori Parata’s whale recovery team

        But indigenous elders say they aren’t being listened to when they tell the government their whale kin are sick, and trying to escape an increasingly polluted and unpredictable ocean.

        Earlier this year in South Taranaki, a mass stranding that was described as “unprecedented” left the local Māori tribe scrambling. Security was brought in when thieves attacked a sperm whale with an axe, trying to remove valuable teeth from its jaw.

        12 Parāoa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory.

        • 12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen near this location in recent memory

        Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te Kaurinui was called after the first whale his father ever named, and left university this year to return to Whangārei and study whale tikanga (protocol) and carving.

        He says mass strandings are getting more local and international attention and money from donations, but traditional knowledge is being dismissed as overly spiritual.

        Kauri (Tekaurinui Robert) Parata, of the New Zealand Māori tribe Ngāti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei

        • Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a beached whale – the middle one shows marks where a poacher had attempted to hack it out with an axe before the recovery group arrived; the Pou, a tribal identifier, in front of the carving shed.

        ‘We need to listen’

        Māori harvest rights over dead whales have only been officially recognised since 1998, and the practice still elicits horror from some New Zealanders and visitors.

        “Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern thing,” says Te Kaurinui.

        The Pou in front of the carving shed at Hihiaua Cultural centre
        Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata.

        The Ngātiwai are investigating a possible link between the crisis of the dieback disease killing New Zealand’s native kauri trees – and threatening the giant Tāne Mahuta, which may be 2,000 years old – and the increase in whale strandings.

        Parata and his family believe whale oil and byproducts could be used to try to cure Kauri dieback, and want more government money and attention directed towards indigenous knowledge of the interconnectedness of the New Zealand environment, and possible indigenous solutions.

        “People dismiss us when we tell them our spiritual understanding of whales – why they are beaching, why they are hurting,” says Te Kaurinui.

        Whangārei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt.

        • Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through the harbourside silt

        “We are not foreigners in this land. We did not take this land off anyone else. We were not lost waiting for some bullheads to tell us what was going on.”

        Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales had stranded on New Zealand shores.

        “I arrived at the beach and we leapfrogged between the animals. They were calling out to each other and reassuring each other,” says Werner. “It was a shock. We’re working to adapt but the ocean is changing so fast.”

        The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whangārei.

        • Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type that the New Zealand Department of Conservation uses for euthanasing stranded whales that are beyond rescue

        A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue.

        The recent spate of mass strandings has been described as “heartbreaking” by the conservation department.

        But for Parata and his family the slow, painful deaths of their ancestors are personal – and ultimately devastating – for the health of the tribe and the sea.

        “It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So what is the sea telling us? We need to listen.”

        ================================================ FILE: test/test-pages/guardian-1/source.html ================================================ 'What is the sea telling us?': Māori tribes fearful over whale strandings | Eleanor Ainge Roy | Environment | The Guardian
        Seascape: the state of our oceans

        'What is the sea telling us?': Māori tribes fearful over whale strandings

        Whale expert Hori Parata. Photograph: Adrian Malloch for the Guardian

        New Zealand’s whale whisperers worry that manmade changes in the ocean are behind the spike in beachings

        by (reporting) and Adrian Malloch (photography)

        Main image: Whale expert Hori Parata. Photograph: Adrian Malloch for the Guardian

        'What is the sea telling us?': Māori tribes fearful over whale strandings

        Whale whisperer Hori Parata was just seven years old when he attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland, their cries screeching through the air on the deserted stretch of sand.

        Seven decades later, Parata, 75, has now overseen more than 500 strandings and is renowned in New Zealand as the leading Māori whale expert, called on by tribes around the country for cultural guidance as marine strandings become increasingly complex and fatal.

        “Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and uncompromising elder of the Ngātiwai tribe of eastern Northland.

        Hori Parata at his Pātaua farm, the place where he was born and grew up.

        • Hori Parata at his Pātaua farm, the place where he was born and grew up

        “We’re having to put up with a lot of stuff today. The public want to hug the whales, they want to touch them, they want to feel good – that’s not the thing. We feel that is ridiculous.”

        Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale stranding capital of the world, with more than 5,000 incidents recorded since 1840, and an average of 300 individual animals beaching themselves each year.

        Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Maōri design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata

        • Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a traditional Māori design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father

        Concrete information on why whales strand remains elusive, but “sickness, navigational error, geographical features, a rapidly falling tide, being chased by a predator, or extreme weather” are all thought to contribute, according to the New Zealand Department of Conservation.

        Climate change is to blame too, scientists think, with warming ocean temperatures moving whales’ prey closer to the shore and forcing them to pursue their food into shallow waters.

        A bin of small whale bones.
        The baleen recovered from a stranded Pygmy Right Whale.
        Squid beaks, from the stomach of a Sperm Whale.

        • Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm whale; the baleen filter-feeder system recovered from a stranded pygmy right whale.

        ‘Unprecedented’ strandings

        November marked the beginning of whale stranding season, and it started with a surge in incidents, according to whale rescue group Project Jonah, with 140 pilot whales beaching and dying on Stewart Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham Islands and a spate of individual cases around the country.

        And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on their flesh – an acute sense of grief is growing among New Zealand’s indigenous people, who regard whales as their ancestors and taonga (treasures).

        “These days it is like a zoo. People just want to come and gawk at us, without even trying to understand what is happening with the animals and the environment,” says Parata, bristling with anger.

        whale strandings

        “When will we talk about what is hurting these animals out on the sea? They are drowning out there, they can’t breathe, they beach themselves to be with the Aunties.”

        Ngātiwai believe the whales beach when they are ready to die and want to return to their families, the Māori people. Then, their human families use the whales’ gift of their bodies for sacred carvings, for traditional medicines, and even for compost.

        There are marked tribal differences across New Zealand and while some tribes work to refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the Department of Conservation and volunteer groups to take the lead in rescue efforts.

        Then the tribe moves in en masse and holds a karakia (prayer), names each animal and sets to work removing their bones, blubber, eyes and teeth for cultural purposes.

        Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata.

        • Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is storing a pair of massive sperm whale jawbones. Cullen is an integral member of Hori Parata’s whale recovery team

        But indigenous elders say they aren’t being listened to when they tell the government their whale kin are sick, and trying to escape an increasingly polluted and unpredictable ocean.

        Earlier this year in South Taranaki, a mass stranding that was described as “unprecedented” left the local Māori tribe scrambling. Security was brought in when thieves attacked a sperm whale with an axe, trying to remove valuable teeth from its jaw.

        12 Parāoa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory.

        • 12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen near this location in recent memory

        Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te Kaurinui was called after the first whale his father ever named, and left university this year to return to Whangārei and study whale tikanga (protocol) and carving.

        He says mass strandings are getting more local and international attention and money from donations, but traditional knowledge is being dismissed as overly spiritual.

        Kauri (Tekaurinui Robert) Parata, of the New Zealand Māori tribe Ngāti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei

        • Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a beached whale – the middle one shows marks where a poacher had attempted to hack it out with an axe before the recovery group arrived; the Pou, a tribal identifier, in front of the carving shed.

        ‘We need to listen’

        Māori harvest rights over dead whales have only been officially recognised since 1998, and the practice still elicits horror from some New Zealanders and visitors.

        “Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern thing,” says Te Kaurinui.

        The Pou in front of the carving shed at Hihiaua Cultural centre
        Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata.

        The Ngātiwai are investigating a possible link between the crisis of the dieback disease killing New Zealand’s native kauri trees – and threatening the giant Tāne Mahuta, which may be 2,000 years old – and the increase in whale strandings.

        Parata and his family believe whale oil and byproducts could be used to try to cure Kauri dieback, and want more government money and attention directed towards indigenous knowledge of the interconnectedness of the New Zealand environment, and possible indigenous solutions.

        “People dismiss us when we tell them our spiritual understanding of whales – why they are beaching, why they are hurting,” says Te Kaurinui.

        Whangārei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt.

        • Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through the harbourside silt

        “We are not foreigners in this land. We did not take this land off anyone else. We were not lost waiting for some bullheads to tell us what was going on.”

        Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales had stranded on New Zealand shores.

        “I arrived at the beach and we leapfrogged between the animals. They were calling out to each other and reassuring each other,” says Werner. “It was a shock. We’re working to adapt but the ocean is changing so fast.”

        The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whangārei.

        • Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type that the New Zealand Department of Conservation uses for euthanasing stranded whales that are beyond rescue

        A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue.

        The recent spate of mass strandings has been described as “heartbreaking” by the conservation department.

        But for Parata and his family the slow, painful deaths of their ancestors are personal – and ultimately devastating – for the health of the tribe and the sea.

        “It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So what is the sea telling us? We need to listen.”

        Topics

        Most popular

        ================================================ FILE: test/test-pages/heise/expected-metadata.json ================================================ { "title": "1Password für Mac generiert Einmal-Passwörter", "byline": "Mac & i", "dir": null, "lang": "de", "excerpt": "Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.", "siteName": "Mac & i", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/heise/expected.html ================================================

        1Password scannt auch QR-Codes.

        (Bild: Hersteller)

        Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.

        AgileBits hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben. Mit dem Update wird eine praktische Funktion nachgereicht, die die iOS-Version der Anwendung bereits seit längerem beherrscht: Das direkte Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der TOTP-Standard (Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.

        Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann – etwa aus dem Webbrowser. Eine Einführung in die Technik gibt ein kurzes Video. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert – was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor mehr gibt.

        Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt worden sein.

        1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen Version mindestens OS X 10.10 voraus. (bsc)

        ================================================ FILE: test/test-pages/heise/source.html ================================================ 1Password für Mac generiert Einmal-Passwörter | Mac & i
        • Mac & i Twitter
        • Mac & i Facebook
        • Mac & i Google+
        • Mac & i YouTube
        • Mac & i Apps
          Mac & i für das iPhone
          Mac & i für das iPad
        Anzeige

        Heft 2/2015

        Billig kontra Apple • Zaubern mit Skripten • HomeKit • Datenrettung • iPad automatisieren • Standort freigeben • Verräterische Daten • Kameras fernsteuern • MacBooks • Bluetooth-Kopfhörer • iPhone-6-Hüllen • Ableton-Live-Apps • Kommt das Apple-Auto?
        Webcode:
        www.mac-and-i.de/

        c't special Mac

        Mac: Der bessere PC • Kaufberatung: Alle Macs im Vergleich • Umsteigen von Windows • 150 Seiten Praxistipps zu OS X und Anwendungen • Mac schneller machen • Erste Hilfe, wenns klemmt • Terminal Mehr…

        Anzeige

        Anzeige

        News

        Apple stopft Sicherheitslücken in iOS und mehreren OS-X-Versionen

        OS X Yosemite

        Apple hat mit den Betriebssystem-Updates vom Mittwoch ein Bündel an Security-Fixes mitgeliefert. Sicherheitsaktualisierungen gibt es auch für OS X 10.8, 10.9 und Apple TV. Mehr…

        Ex-HP-Chefin kritisiert Tim Cooks Kritik an Gesetz zur Religionsfreiheit

        Ehemalige HP-Chefin kritisiert Tim Cooks Kritik an Gesetz zur Religionsfreiheit

        Der Apple-Chef hatte sich gegen die Gesetzgebung eines US-Bundesstaates ausgesprochen, der die Diskriminierung von Schwulen und Lesben unter bestimmten Umständen erlauben soll. Carly Fiorina hält dies für "heuchlerisch". Mehr…

        Apple Watch: Bestellungen können offenbar dauern

        Apple Watch: Bestellungen können offenbar dauern

        Im deutschen und im britischen Apple Online Store sind kurzzeitig Liefertermine für die Computeruhr aufgetaucht. Manche Modelle brauchen demnach bis zu sechs Wochen. Mehr…

        Apple aktualisiert Xcode, OS X Server und Apple Configurator

        Apple aktualisiert Xcode, OS X Server und Apple Configurator

        Neben iOS 8.3 und OS X 10.10.3 hat der Hersteller auch drei weitere hauseigene Anwendungen auf den neuesten Stand gebracht. Xcode 6.3 bringt unter anderem Support für die neue Force-Touch-Technik und ein neues Swift. Mehr…

        Quiz: Wie gut kennen Sie Apple?

        Quiz: Wie gut kennen Sie Apple?

        12 Fragen rund um das Unternehmen und seine Produkte – einige davon dürften nur Fortgeschrittene knacken können. Mehr…

        Artikel

        Pro & Contra: Hat Apple den Bogen überspannt?

        Beim neuen MacBook 12" gibt es außer der Kopfhörerbuchse nur eine Schnittstelle: USB Typ C. Darüber scheiden sich wieder mal die Geister. Mehr…

        Praxistipp: Größere SSD im MacBook Air 2011

        Wenn das MacBook an seine Grenzen stößt, brauchen Sie nicht unbedingt ein neues: Ersatz-SSDs gibt es ab 170 Euro, der Umbau ist auch für Laien zu schaffen und in wenigen Minuten erledigt. Mehr…

        Pro & Contra: Ist Apple zu streng?

        Das Prüferteam im iOS-App-Store lehnt immer häufiger ganze Apps ab oder verlangt eine Beschneidung der Funktionen. Ist das richtig so? Mehr…

        Die besseren Mac minis

        Nachdem wir den günstigsten Mac mini mit 1,4 GHz bereits in Mac & i Heft 6/2014 vorstellen konnten, reichen wir nun – wie versprochen – einen Test der beiden besseren Konfigurationen nach. Mehr…

        In eigener Sache: Mac & i im Digitalabo

        In eigener Sache: Mac &amp; i im Digitalabo

        Heise Medien bietet Lesern, die kein gedrucktes Heft mehr wollen, nun auch ein vergünstigtes Digitalabo von Mac & i für das iPad an. Mehr…

        ================================================ FILE: test/test-pages/herald-sun-1/expected-metadata.json ================================================ { "title": "Angry media won’t buckle over new surveillance laws", "byline": "JOE HILDEBRAND", "dir": null, "lang": "en-au", "excerpt": "A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed.", "siteName": "HeraldSun", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/herald-sun-1/expected.html ================================================

        A new Bill would require telecommunications service providers to store so-called ‘metadat

        A new Bill would require telecommunications service providers to store so-called ‘metadata’ for two years. Source: Supplied

        A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed.

        The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer, Justin Bassi, who advises Attorney-General George Brandis on crime and security matters, and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of Communications Minister Malcolm Turnbull also took part.

        They held meetings with executives from News Corporation and Fairfax, representatives of the TV networks, the ABC top brass and a group from the media union and the Walkley journalism foundation. I was involved as a member of the Walkley board.

        The initiative, from Tony Abbott’s office, is evidence that the Government has been alarmed by the strength of criticism from media of the Data Retention Bill it wants passed before Parliament rises in a fortnight. Bosses, journalists, even the Press Council, are up in arms, not only over this measure, but also over aspects of two earlier pieces of national security legislation that interfere with the ability of the media to hold government to account.

        The Bill would require telecommunications service providers to store so-called “metadata” — the who, where, when and how of a communication, but not its content — for two years so security and law enforcement agencies can access it without warrant. Few would argue against the use of such material to catch criminals or terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has pointed out, it would also be used “for the purpose of determining the identity of a journalist’s sources”.

        And that should ring warning bells for anyone genuinely concerned with the health of our democracy. Without the ability to protect the identity of sources, journalists would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence and misbehaviour by public officials.

        The Press Council is concerned the laws would crush investigative journalism.

        “These legitimate concerns cannot be addressed effectively short of exempting journalists and media organisations,” says president David Weisbrot.

        The media union is adamant journalists’ metadata must be exempted from the law. That’s what media bosses want, too, though they have a fallback position based on new safeguards being implemented in Britain.

        That would prevent access to the metadata of journalists or media organisations without a judicial warrant. There would be a code including — according to the explanatory notes of the British Bill — “provision to protect the public interest in the confidentiality of journalistic sources”.

        In their meetings this week, the government team boasted of concessions in the new Data Retention Bill. The number of agencies able to access metadata will be reduced by excluding such organisations as the RSPCA and local councils. And whenever an authorisation is issued for access to information about a journalist’s sources, the Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and Security) will receive a copy.

        That does nothing to solve the problem. The Government has effectively admitted as much by agreeing that the parliamentary committee should conduct a separate review of how to deal with the issue of journalists’ sources.

        But another inquiry would be a waste of time — the committee has already received and considered dozens of submissions on the subject. The bottom line is that the Government does not deny that the legislation is flawed, but is demanding it be passed anyway with the possibility left open of a repair job down the track. That is a ridiculous approach.

        Claims that immediate action is imperative do not stand up. These are measures that won’t come into full effect for two years. Anyway, amending the Bill to either exempt journalists or adopt the UK model could be done quickly, without any risk to national security.

        AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press freedom concerns about mandatory data retention would ideally be addressed in this Bill to avoid the need for future additional amendments or procedures to be put in place in the future.”

        The Data Retention Bill will be debated in the House of Representatives this week. Then, on Friday, CEOs from leading media organisations will front the parliamentary committee to air their concerns before the legislation goes to the Senate.

        Those CEOs should make it clear they are just as angry about this as they were about Stephen Conroy’s attempt to impinge on press freedom through media regulation under the previous Labor government.

        Memories of the grief Conroy brought down on his head would undoubtedly make Abbott sit up and take notice.

        LAURIE OAKES IS THE NINE NETWORK POLITICAL EDITOR

        ================================================ FILE: test/test-pages/herald-sun-1/source.html ================================================ Angry media won’t buckle over new surveillance laws | Herald Sun

        Laurie Oakes

        Angry media won’t buckle over new surveillance laws

        • by: Laurie Oakes
        • From: Herald Sun
        • March 13, 2015 9:00PM
        A new Bill would require telecommunications service providers to store so-called ‘metadat

        A new Bill would require telecommunications service providers to store so-called ‘metadata’ for two years. Source: Supplied

        A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed.

        The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer, Justin Bassi, who advises Attorney-General George Brandis on crime and security matters, and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of Communications Minister Malcolm Turnbull also took part.

        They held meetings with executives from News Corporation and Fairfax, representatives of the TV networks, the ABC top brass and a group from the media union and the Walkley journalism foundation. I was involved as a member of the Walkley board.

        The initiative, from Tony Abbott’s office, is evidence that the Government has been alarmed by the strength of criticism from media of the Data Retention Bill it wants passed before Parliament rises in a fortnight. Bosses, journalists, even the Press Council, are up in arms, not only over this measure, but also over aspects of two earlier pieces of national security legislation that interfere with the ability of the media to hold government to account.

        The Bill would require telecommunications service providers to store so-called “metadata” — the who, where, when and how of a communication, but not its content — for two years so security and law enforcement agencies can access it without warrant. Few would argue against the use of such material to catch criminals or terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has pointed out, it would also be used “for the purpose of determining the identity of a journalist’s sources”.

        And that should ring warning bells for anyone genuinely concerned with the health of our democracy. Without the ability to protect the identity of sources, journalists would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence and misbehaviour by public officials.

        The Press Council is concerned the laws would crush investigative journalism.

        “These legitimate concerns cannot be addressed effectively short of exempting journalists and media organisations,” says president David Weisbrot.

        The media union is adamant journalists’ metadata must be exempted from the law. That’s what media bosses want, too, though they have a fallback position based on new safeguards being implemented in Britain.

        That would prevent access to the metadata of journalists or media organisations without a judicial warrant. There would be a code including — according to the explanatory notes of the British Bill — “provision to protect the public interest in the confidentiality of journalistic sources”.

        In their meetings this week, the government team boasted of concessions in the new Data Retention Bill. The number of agencies able to access metadata will be reduced by excluding such organisations as the RSPCA and local councils. And whenever an authorisation is issued for access to information about a journalist’s sources, the Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and Security) will receive a copy.

        That does nothing to solve the problem. The Government has effectively admitted as much by agreeing that the parliamentary committee should conduct a separate review of how to deal with the issue of journalists’ sources.

        But another inquiry would be a waste of time — the committee has already received and considered dozens of submissions on the subject. The bottom line is that the Government does not deny that the legislation is flawed, but is demanding it be passed anyway with the possibility left open of a repair job down the track. That is a ridiculous approach.

        Claims that immediate action is imperative do not stand up. These are measures that won’t come into full effect for two years. Anyway, amending the Bill to either exempt journalists or adopt the UK model could be done quickly, without any risk to national security.

        AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press freedom concerns about mandatory data retention would ideally be addressed in this Bill to avoid the need for future additional amendments or procedures to be put in place in the future.”

        The Data Retention Bill will be debated in the House of Representatives this week. Then, on Friday, CEOs from leading media organisations will front the parliamentary committee to air their concerns before the legislation goes to the Senate.

        Those CEOs should make it clear they are just as angry about this as they were about Stephen Conroy’s attempt to impinge on press freedom through media regulation under the previous Labor government.

        Memories of the grief Conroy brought down on his head would undoubtedly make Abbott sit up and take notice.

        LAURIE OAKES IS THE NINE NETWORK POLITICAL EDITOR

        more stories

        Other Opinion Columns

        Editorial Fraser’s lasting legacy

        Fraser’s lasting legacy

        MALCOLM Fraser — an effective prime minister controversially installed, soundly elected and re-elected, a relentless contributor, thinker and reformer, a blue-blooded egalitarian.

        Wave of stupidity Doctor knows best, not Starbeam Fantapants

        Doctor knows best, not Starbeam Fantapants

        JOE HILDEBRAND HOW is it that ideas and philosophies that are patently absurd capture people’s imagination? It seems we’re being swamped by a wave of stupidity.

        Ikea challenge Good for some, but I’m not buying it

        Good for some, but I’m not buying it

        SUSIE O’BRIEN THE good thing about Ikea is that it’s cheap and good quality. The bad thing is the whole, exhausting business of shopping at their stores.

        True liberal Ex-PM a man of contrasts

        Ex-PM a man of contrasts

        LAURIE OAKES IT was said that Malcolm Fraser moved to the Left after losing office but the former PM didn’t change - he was always a “small l” liberal.

        Own backyard Let’s make local issues a priority

        Let’s make local issues a priority

        TOM ELLIOTT IT’S nice to think Australia is a player on the world stage but, really, we’re not. So forget global posturing and let’s sort out our own backyard.

        Most Popular Stories

        1. Wife’s fury at hubby’s million dollar gamble
        2. Australia passes fierce Pakistan test
        3. Masa’s brave dad faces accused killer
        4. Police allege Price went on 90-minute rampage
        5. This time Cousins really dodged a bullet
        ================================================ FILE: test/test-pages/hidden-nodes/expected-metadata.json ================================================ { "title": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt", "byline": null, "dir": null, "lang": "en", "excerpt": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/hidden-nodes/expected.html ================================================

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Third header

        ================================================ FILE: test/test-pages/hidden-nodes/source.html ================================================ Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Third header

        ================================================ FILE: test/test-pages/hukumusume/expected-metadata.json ================================================ { "title": "欲張りなイヌ　＜福娘童話集　きょうのイソップ童話＞", "byline": null, "dir": null, "excerpt": "福娘童話集 > きょうのイソップ童話 > １月のイソップ童話 > 欲張りなイヌ", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/hukumusume/expected.html ================================================
         

        福娘童話集 > きょうのイソップ童話 > １月のイソップ童話 > 欲張りなイヌ

        元旦のイソップ童話

        よくばりなイヌ

        欲張りなイヌ

        ひらがな　←→　日本語・英語　←→　English

        おりがみをつくろう ( おりがみくらぶ より)

        犬の顔の折り紙いぬのかお　　　犬の顔の紙いぬ

        ♪音声配信(html5)
        亜姫の朗読☆　イソップ童話より

        　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。

        おしまい

        前のページへ戻る
             1月 1日の豆知識

        366日への旅
        きょうの記念日元旦
        きょうの誕生花松(まつ)
        きょうの誕生日・出来事1949年　Mr.マリック(マジシャン)
        恋の誕生日占い自分の考えをしっかりと持った女の子。
        なぞなぞ小学校○(丸)を取ったらお母さんになってしまう男の人は？
        あこがれの職業紹介歌手
        恋の魔法とおまじない 001両思いになれる おまじない
        　　1月 1日の童話・昔話

        福娘童話集
        きょうの日本昔話ネコがネズミを追いかける訳
        きょうの世界昔話モンゴルの十二支話
        きょうの日本民話仕事の取替えっこ
        きょうのイソップ童話欲張りなイヌ
        きょうの江戸小話ぞうきんとお年玉
        きょうの百物語百物語の幽霊
        福娘のサイト
        366日への旅

        毎日の記念日・誕生花 ・有名人の誕生日と性格判断

        福娘童話集

        世界と日本の童話と昔話

        女の子応援サイト -さくら-

        誕生日占い、お仕事紹介、おまじない、など

        子どもの病気相談所

        病気検索と対応方法、症状から検索するWEB問診

        世界60秒巡り

        国旗国歌や世界遺産など、世界の国々の豆知識

        ================================================ FILE: test/test-pages/hukumusume/source.html ================================================ 欲張りなイヌ　＜福娘童話集　きょうのイソップ童話＞
         

        福娘童話集 > きょうのイソップ童話 > １月のイソップ童話 > 欲張りなイヌ

        元旦のイソップ童話



        よくばりなイヌ



        欲張りなイヌ



        ひらがな　←→　日本語・英語　←→　English

        おりがみをつくろう ( おりがみくらぶ より)
        犬の顔の折り紙いぬのかお　　　犬の顔の紙いぬ
        ♪音声配信(html5)
        亜姫の朗読☆　イソップ童話より

        　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。

        おしまい

        前のページへ戻る
             1月 1日の豆知識



        366日への旅
        きょうの記念日

        元旦
        きょうの誕生花

        松(まつ)
        きょうの誕生日・出来事

        1949年　Mr.マリック(マジシャン)
        恋の誕生日占い

        自分の考えをしっかりと持った女の子。
        なぞなぞ小学校

        ○(丸)を取ったらお母さんになってしまう男の人は？
        あこがれの職業紹介

        歌手
        恋の魔法とおまじない 001

        両思いになれる おまじない
        　　1月 1日の童話・昔話



        福娘童話集
        きょうの日本昔話

        ネコがネズミを追いかける訳
        きょうの世界昔話

        モンゴルの十二支話
        きょうの日本民話

        仕事の取替えっこ
        きょうのイソップ童話

        欲張りなイヌ
        きょうの江戸小話

        ぞうきんとお年玉
        きょうの百物語

        百物語の幽霊
        福娘のサイト
        366日への旅

        毎日の記念日・誕生花 ・有名人の誕生日と性格判断
        福娘童話集

        世界と日本の童話と昔話
        女の子応援サイト -さくら-

        誕生日占い、お仕事紹介、おまじない、など
        子どもの病気相談所

        病気検索と対応方法、症状から検索するWEB問診
        世界60秒巡り

        国旗国歌や世界遺産など、世界の国々の豆知識
        ================================================ FILE: test/test-pages/iab-1/expected-metadata.json ================================================ { "title": "Getting LEAN with Digital Ad UX | IAB", "byline": "By\n\t\t\tScott Cunningham", "dir": null, "lang": "en-US", "excerpt": "We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience. Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. … Continued", "siteName": "IAB", "publishedTime": "2015-10-15T08:00:26+00:00", "readerable": true } ================================================ FILE: test/test-pages/iab-1/expected.html ================================================

        We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience.

        Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. Many of us in the technical field felt compelled, and even empowered, to produce information as the distribution means for mass communication were no longer restricted by a high barrier to entry.

        In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups were gone or that our divisions sustained by corporate parent companies needed to be in the black. It was a wakeup call that led to a renaissance age. Digital advertising became the foundation of an economic engine that, still now, sustains the free and democratic World Wide Web. In digital publishing, we strived to balance content, commerce, and technology. The content management systems and communication gateways we built to inform and entertain populations around the world disrupted markets and in some cases governments, informed communities of imminent danger, and liberated new forms of art and entertainment—all while creating a digital middle class of small businesses.

        We engineered not just the technical, but also the social and economic foundation that users around the world came to lean on for access to real time information. And users came to expect this information whenever and wherever they needed it. And more often than not, for anybody with a connected device, it was free.

        This was choice—powered by digital advertising—and premised on user experience.

        But we messed up.

        Through our pursuit of further automation and maximization of margins during the industrial age of media technology, we built advertising technology to optimize publishers’ yield of marketing budgets that had eroded after the last recession. Looking back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast, scalable systems of targeting users with ever-heftier advertisements have slowed down the public internet and drained more than a few batteries. We were so clever and so good at it that we over-engineered the capabilities of the plumbing laid down by, well, ourselves. This steamrolled the users, depleted their devices, and tried their patience.

        The rise of ad blocking poses a threat to the internet and could potentially drive users to an enclosed platform world dominated by a few companies. We have let the fine equilibrium of content, commerce, and technology get out of balance in the open web. We had, and still do have, a responsibility to educate the business side, and in some cases to push back. We lost sight of our social and ethical responsibility to provide a safe, usable experience for anyone and everyone wanting to consume the content of their choice.

        We need to bring that back into alignment, starting right now.

        Getting LEAN with Digital Ad UXToday, the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads. These are principles that will help guide the next phases of advertising technical standards for the global digital advertising supply chain.

        As with any other industry, standards should be created by non-profit standards-setting bodies, with many diverse voices providing input. We will invite all parties for public comment, and make sure consumer interest groups have the opportunity to provide input.

        L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy and engage with while consuming content on our sites across all IP enabled devices. Rather, these principles will guide an alternative set of standards that provide choice for marketers, content providers, and consumers.

        Among the many areas of concentration, we must also address frequency capping on retargeting in Ad Tech and make sure a user is targeted appropriately before, but never AFTER they make a purchase. If we are so good at reach and scale, we can be just as good, if not better, at moderation. Additionally, we must address volume of ads per page as well as continue on the path to viewability. The dependencies here are critical to an optimized user experience.

        The consumer is demanding these actions, challenging us to do better, and we must respond.

        The IAB Tech Lab will continue to provide the tools for publishers in the digital supply chain to have a dialogue with users about their choices so that content providers can generate revenue while creating value. Publishers should have the opportunity to provide rich advertising experiences, L.E.A.N. advertising experiences, and subscription services. Or publishers can simply deny their service to users who choose to keep on blocking ads. That is all part of elasticity of consumer tolerance and choice.

        Finally, we must do this in an increasingly fragmented market, across screens. We must do this in environments where entire sites are blocked, purposefully or not. Yes, it is disappointing that our development efforts will have to manage with multiple frameworks while we work to supply the economic engine to sustain an open internet. However, our goal is still to provide diverse content and voices to as many connected users as possible around the world.

        That is user experience.

        IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email adblocking@iab.com for more information.

        Read more about ad blocking here.

        Auto Draft 14

        About the author

        Scott Cunningham

        Senior Vice President of Technology and Ad Operations at IAB, and General Manager of the IAB Tech Lab

        ================================================ FILE: test/test-pages/iab-1/source.html ================================================ Getting LEAN with Digital Ad UX | IAB arrow-downarrow-leftarrow-rightarrow-upbiocircleclosedownloadfacebookgplusinstagramlinkedinmailmenuphoneplaysearchsharespinnertwitteryoutube

        Getting LEAN with Digital Ad UX

        By Scott Cunningham
        Getting LEAN with Digital Ad UX 1

        We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience.

        Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. Many of us in the technical field felt compelled, and even empowered, to produce information as the distribution means for mass communication were no longer restricted by a high barrier to entry.

        In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups were gone or that our divisions sustained by corporate parent companies needed to be in the black. It was a wakeup call that led to a renaissance age. Digital advertising became the foundation of an economic engine that, still now, sustains the free and democratic World Wide Web. In digital publishing, we strived to balance content, commerce, and technology. The content management systems and communication gateways we built to inform and entertain populations around the world disrupted markets and in some cases governments, informed communities of imminent danger, and liberated new forms of art and entertainment—all while creating a digital middle class of small businesses.

        We engineered not just the technical, but also the social and economic foundation that users around the world came to lean on for access to real time information. And users came to expect this information whenever and wherever they needed it. And more often than not, for anybody with a connected device, it was free.

        This was choice—powered by digital advertising—and premised on user experience.

        But we messed up.

        Through our pursuit of further automation and maximization of margins during the industrial age of media technology, we built advertising technology to optimize publishers’ yield of marketing budgets that had eroded after the last recession. Looking back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast, scalable systems of targeting users with ever-heftier advertisements have slowed down the public internet and drained more than a few batteries. We were so clever and so good at it that we over-engineered the capabilities of the plumbing laid down by, well, ourselves. This steamrolled the users, depleted their devices, and tried their patience.

        The rise of ad blocking poses a threat to the internet and could potentially drive users to an enclosed platform world dominated by a few companies. We have let the fine equilibrium of content, commerce, and technology get out of balance in the open web. We had, and still do have, a responsibility to educate the business side, and in some cases to push back. We lost sight of our social and ethical responsibility to provide a safe, usable experience for anyone and everyone wanting to consume the content of their choice.

        We need to bring that back into alignment, starting right now.

        Getting LEAN with Digital Ad UXToday, the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads. These are principles that will help guide the next phases of advertising technical standards for the global digital advertising supply chain.

        As with any other industry, standards should be created by non-profit standards-setting bodies, with many diverse voices providing input. We will invite all parties for public comment, and make sure consumer interest groups have the opportunity to provide input.

        L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy and engage with while consuming content on our sites across all IP enabled devices. Rather, these principles will guide an alternative set of standards that provide choice for marketers, content providers, and consumers.

        Among the many areas of concentration, we must also address frequency capping on retargeting in Ad Tech and make sure a user is targeted appropriately before, but never AFTER they make a purchase. If we are so good at reach and scale, we can be just as good, if not better, at moderation. Additionally, we must address volume of ads per page as well as continue on the path to viewability. The dependencies here are critical to an optimized user experience.

        The consumer is demanding these actions, challenging us to do better, and we must respond.

        The IAB Tech Lab will continue to provide the tools for publishers in the digital supply chain to have a dialogue with users about their choices so that content providers can generate revenue while creating value. Publishers should have the opportunity to provide rich advertising experiences, L.E.A.N. advertising experiences, and subscription services. Or publishers can simply deny their service to users who choose to keep on blocking ads. That is all part of elasticity of consumer tolerance and choice.

        Finally, we must do this in an increasingly fragmented market, across screens. We must do this in environments where entire sites are blocked, purposefully or not. Yes, it is disappointing that our development efforts will have to manage with multiple frameworks while we work to supply the economic engine to sustain an open internet. However, our goal is still to provide diverse content and voices to as many connected users as possible around the world.

        That is user experience.

         

        IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email adblocking@iab.com for more information.

        Read more about ad blocking here.

        Related Content

        Show more related content
        ================================================ FILE: test/test-pages/ietf-1/expected-metadata.json ================================================ { "title": "remoteStorage", "byline": "Jong, Michiel de", "dir": null, "lang": "en", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/ietf-1/expected.html ================================================
        [Docs] [txt|pdf] [Tracker] [Email] [Diff1] [Diff2] [Nits]

        Versions: 00 01 02 03 04 

        INTERNET DRAFT                                      Michiel B. de Jong
Document: draft-dejong-remotestorage-04                   IndieHosters
                                                             F. Kooman
Intended Status: Proposed Standard                       (independent)
Expires: 18 June 2015                                 15 December 2014


                            remoteStorage

Abstract

    This draft describes a protocol by which client-side applications,
    running inside a web browser, can communicate with a data storage
    server that is hosted on a different domain name. This way, the
    provider of a web application need not also play the role of data
    storage provider. The protocol supports storing, retrieving, and
    removing individual documents, as well as listing the contents of an
    individual folder, and access control is based on bearer tokens.

Status of this Memo

   This Internet-Draft is submitted in full conformance with the
   provisions of BCP 78 and BCP 79.

   Internet-Drafts are working documents of the Internet Engineering
   Task Force (IETF).  Note that other groups may also distribute
   working documents as Internet-Drafts.  The list of current Internet-
   Drafts is at http://datatracker.ietf.org/drafts/current/.

   Internet-Drafts are draft documents valid for a maximum of six months
   and may be updated, replaced, or obsoleted by other documents at any
   time.  It is inappropriate to use Internet-Drafts as reference
   material or to cite them other than as "work in progress."

   This Internet-Draft will expire on 15 December 2014.

Copyright Notice

   Copyright (c) 2014 IETF Trust and the persons identified as the
   document authors. All rights reserved.

   This document is subject to BCP 78 and the IETF Trust's Legal
   Provisions Relating to IETF Documents
   (http://trustee.ietf.org/license-info) in effect on the date of
   publication of this document.  Please review these documents
   carefully, as they describe your rights and restrictions with respect
   to this document.  Code Components extracted from this document must
   include Simplified BSD License text as described in Section 4.e of
   the Trust Legal Provisions and are provided without warranty as
   described in the Simplified BSD License.


de Jong                                                         [Page 1]

         
Internet-Draft              remoteStorage                  December 2014


Table of Contents

   1. Introduction...................................................2
   2. Terminology....................................................3
   3. Storage model..................................................3
   4. Requests.......................................................4
   5. Response codes.................................................7
   6. Versioning.....................................................7
   7. CORS headers...................................................8
   8. Session description............................................8
   9. Bearer tokens and access control...............................9
  10. Application-first bearer token issuance.......................10
  11. Storage-first bearer token issuance...........................11
  12. Example wire transcripts......................................12
     12.1. WebFinger................................................12
     12.2. OAuth dialog form........................................13
     12.3. OAuth dialog form submission.............................14
     12.4. OPTIONS preflight........................................15
     12.5. Initial PUT..............................................15
     12.6. Subsequent PUT...........................................16
     12.7. GET......................................................16
     12.8. DELETE...................................................17
  13. Distributed versioning........................................17
  14. Security Considerations.......................................19
  15. IANA Considerations...........................................20
  16. Acknowledgments...............................................20
  17. References....................................................21
     17.1. Normative References.....................................21
     17.2. Informative References...................................21
  18. Authors' addresses............................................22


1.  Introduction

    Many services for data storage are available over the internet. This
    specification describes a vendor-independent interface for such
    services. It is based on https, CORS and bearer tokens. The
    metaphor for addressing data on the storage is that of folders
    containing documents and subfolders. The actions the interface
    exposes are:

       *  GET a folder: retrieve the names and current versions of the
          documents and subfolders currently contained by the folder


de Jong                                                         [Page 2]

         
Internet-Draft              remoteStorage                  December 2014


       *  GET a document: retrieve its content type, current version,
          and contents

       *  PUT a document: store a new version, its content type, and
          contents, conditional on the current version

       *  DELETE a document: remove it from the storage, conditional on
          the current version

       *  HEAD a folder or document: like GET, but omitting the response
          body

    The exact details of these four actions are described in this
    specification.

2. Terminology

    The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
    "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
    document are to be interpreted as described in RFC 2119 [WORDS].

    "SHOULD" and "SHOULD NOT" are appropriate when valid exceptions to a
    general requirement are known to exist or appear to exist, and it is
    infeasible or impractical to enumerate all of them.  However, they
    should not be interpreted as permitting implementors to fail to
    implement the general requirement when such failure would result in
    interoperability failure.

3. Storage model

    The server stores data in nodes that form a tree structure.
    Internal nodes are called 'folders' and leaf nodes are called
    'documents'. For a folder, the server stores references to nodes
    contained in the folder, and it should be able to produce a list of
    them, with for each contained item:

       * item name
       * item type (folder or document)
       * current version
       * content type
       * content length

    For a document, the server stores, and should be able to produce:


de Jong                                                         [Page 3]

         
Internet-Draft              remoteStorage                  December 2014



       * current version
       * content type
       * content length
       * content

4. Requests

    Client-to-server requests SHOULD be made over https [HTTPS], and
    servers MUST comply with HTTP/1.1 [HTTP]. Specifically, they
    MUST support chunked transfer coding on PUT requests. Servers MAY
    also offer an optional switch from https to SPDY [SPDY].

    A request is considered successful if the HTTP response code is in
    the 2xx range (e.g. 200 OK, 201 Created), and unsuccessful if an
    error occurred or a condition was not met (response code e.g. 404
    Not Found, 304 Not Modified).

    The root folder of the storage tree is represented by the following
    URL:

        URI_ENCODE( <storage_root> '/' )

    Subsequently, if <parent_folder> is the URL of a folder, then the
    URL of an item contained in it is:

        URI_ENCODE( <parent_folder> <document_name> )

    for a document, or:

        URI_ENCODE( <parent_folder> <folder_name> '/' )

    for a folder. Item names MAY contain all characters except '/' and
    the null character, and MUST NOT have zero length.

    A document description is a map containing one string-valued 'ETag'
    field, one string-valued 'Content-Type' and one integer-valued
    'Content-Length' field. They represent the document's current
    version, its content type, and its content length respectively. Note
    that content length is measured in octets (bytes), not in
    characters.

    A folder description is a map containing a string-valued 'ETag'


de Jong                                                         [Page 4]

         
Internet-Draft              remoteStorage                  December 2014


    field, representing the folder's current version.

    A successful GET request to a folder MUST be responded to with a
    JSON-LD [JSON-LD] document (content type 'application/ld+json'),
    containing as its 'items' field a map in which contained documents
    appear as entries <item_name> to a document description, and
    contained non-empty folders appear as entries <item_name> '/' to a
    folder description. It MUST also contain an '@context' field with
    the value 'http://remotestorage.io/spec/folder-description'. For
    instance:

       {
         "@context": "http://remotestorage.io/spec/folder-description",
         "items": {
           "abc": {
             "ETag": "DEADBEEFDEADBEEFDEADBEEF",
             "Content-Type": "image/jpeg",
             "Content-Length": 82352
           },
           "def/": {
             "ETag": "1337ABCD1337ABCD1337ABCD"
           }
         }
       }

    All folders are treated as existing, and therefore GET requests to
    untouched folders SHOULD be responded to with a folder description
    with no items (the items field set to '{}'). However, an empty
    folder MUST NOT be listed as an item in its parent folder.

    Also, since folders exist automatically, PUT and DELETE requests
    only need to be made to documents, and never to folders. A document
    PUT will make all ancestor folders along its path become non-empty;
    deleting the last document from a subtree will make that whole
    subtree become empty. Folders will therefore show up in their parent
    folder descriptions if and only if their subtree contains at least
    one document.

    A successful GET request to a document SHOULD be responded to with
    the full document contents in the body, the document's content type
    in a 'Content-Type' header, its content length in octets (not in
    characters) in a 'Content-Length' header, and the document's current
    version as a strong ETag in an 'ETag' header.


de Jong                                                         [Page 5]

         
Internet-Draft              remoteStorage                  December 2014



    Note that the use of strong ETags prohibits changing the response
    body based on request headers; in particular, the server will not be
    able to serve the same document uncompressed to some clients and
    gzipped when requested by the client, since the two bodies would not
    be identical byte-for-byte.

    Servers MAY support Content-Range headers [RANGE] on GET requests,
    but whether or not they do SHOULD be announced through the <ranges>
    variable mentioned below in section 10.

    A successful PUT request to a document MUST result in:

       * the request body being stored as the document's new content,
       * parent and further ancestor folders being silently created as
         necessary, with the document (name and version) being added to
         its parent folder, and each folder added to its subsequent
         parent,
       * the value of its Content-Type header being stored as the
         document's new content type,
       * its version being updated, as well as that of its parent folder
         and further ancestor folders, using a strong validator [HTTP,
         section 7.2].

    The response MUST contain a strong ETag header, with the document's
    new version (for instance a hash of its contents) as its value.

    A successful DELETE request to a document MUST result in:

       * the deletion of that document from the storage, and from its
         parent folder,
       * silent deletion of the parent folder if it is left empty by
         this, and so on for further ancestor folders,
       * the version of its parent folder being updated, as well as that
         of further ancestor folders.

    A successful OPTIONS request SHOULD be responded to as described in
    the CORS section below.

    A successful HEAD request SHOULD be responded to like to the
    equivalent GET request, but omitting the response body.




de Jong                                                         [Page 6]

         
Internet-Draft              remoteStorage                  December 2014


5. Response codes

    Response codes SHOULD be given as defined by [HTTP, section 6] and
    [BEARER, section 3.1]. The following is a non-normative checklist
    of status codes that are likely to occur in practice:

       * 500 if an internal server error occurs,
       * 429 if the client makes too frequent requests or is suspected
             of malicious activity,
       * 414 if the request URI is too long,
       * 416 if Range requests are supported by the server and the Range
             request can not be satisfied,
       * 401 for all requests that don't have a bearer token with
             sufficient permissions,
       * 404 for all DELETE and GET requests to documents that do not
             exist on the storage,
       * 304 for a conditional GET request whose pre-condition
             fails (see "Versioning" below),
       * 409 for a PUT request where any folder name in the path
             clashes with an existing document's name at the same
             level, or where the document name coincides with an
             existing folder's name at the same level.
       * 412 for a conditional PUT or DELETE request whose pre-condition
             fails (see "Versioning" below),
       * 507 in case the account is over its storage quota,
       * 4xx for all malformed requests (e.g. foreign characters in the
             path), as well as for all PUT and DELETE requests to
             folders,
       * 2xx for all successful requests.

    Clients SHOULD also handle the case where a response takes too long
    to arrive, or where no response is received at all.

6. Versioning

    All successful requests MUST return an 'ETag' header [HTTP] with, in
    the case of GET, the current version, in the case of PUT, the new
    version, and in case of DELETE, the version that was deleted. All
    successful GET requests MUST return an 'Expires: 0' header. PUT and
    DELETE requests MAY have an 'If-Match' request header [COND], and
    MUST fail with a 412 response code if that doesn't match the
    document's current version.



de Jong                                                         [Page 7]

         
Internet-Draft              remoteStorage                  December 2014


    GET requests MAY have a comma-separated list of revisions in an
    'If-None-Match' header [COND], and SHOULD be responded to with a 304
    response if that list includes the document or folder's current
    version. A PUT request MAY have an 'If-None-Match: *' header [COND],
    in which case it MUST fail with a 412 response code if the document
    already exists.

    In all 'ETag', 'If-Match' and 'If-None-Match' headers, revision
    strings should appear inside double quotes (").

    A provider MAY offer version rollback functionality to its users,
    but this specification does not define the user interface for that.

7. CORS headers

    All responses MUST carry CORS headers [CORS]. The server MUST also
    reply to OPTIONS requests as per CORS. For GET requests, a wildcard
    origin MAY be returned, but for PUT and DELETE requests, the
    response MUST echo back the Origin header sent by the client.

8. Session description

    The information that a client needs to receive in order to be able
    to connect to a server SHOULD reach the client as described in the
    'bearer token issuance' sections below. It consists of:

       * <storage_root>, consisting of 'https://' followed by a server
         host, and optionally a server port and a path prefix as per
         [IRI]. Examples:
         * 'https://example.com' (host only)
         * 'https://example.com:8080' (host and port)
         * 'https://example.com/path/to/storage' (host, port and
           path prefix; note there is no trailing slash)
       * <access_token> as per [OAUTH]. The token SHOULD be hard to
         guess and SHOULD NOT be reused from one client to another. It
         can however be reused in subsequent interactions with the same
         client, as long as that client is still trusted. Example:
         * 'ofb24f1ac3973e70j6vts19qr9v2eei'
       * <storage_api>, always 'draft-dejong-remotestorage-04' for this
         alternative version of the specification.

    The client can make its requests using https with CORS and bearer
    tokens, to the URL that is the concatenation of <storage_root> with


de Jong                                                         [Page 8]

         
Internet-Draft              remoteStorage                  December 2014


    '/' plus one or more <folder> '/' strings indicating a path in the
    folder tree, followed by zero or one <document> strings, indicating
    a document. For example, if <storage_root> is
    "https://storage.example.com/bob", then to retrieve the folder
    contents of the /public/documents/ folder, or to retrieve a
    'draft.txt' document from that folder, the client would make
    requests to, respectively:

    * https://storage.example.com/bob/public/documents/
    * https://storage.example.com/bob/public/documents/draft.txt

9. Bearer tokens and access control

    A bearer token represents one or more access scopes. These access
    scopes are represented as strings of the form <module> <level>,
    where the <module> string SHOULD be lower-case alphanumerical, other
    than the reserved word 'public', and <level> can be ':r' or ':rw'.
    The access the bearer token gives is the sum of its access scopes,
    with each access scope representing the following permissions:

    '*:rw') any request,

    '*:r') any GET or HEAD request,

    <module> ':rw') any requests to paths that start with
                    '/' <module> '/' or '/public/' <module> '/',

    <module> ':r') any GET or HEAD requests to paths that start with
                   '/' <module> '/' or '/public/' <module> '/',

    As a special exceptions, GET requests to a document (but not a
    folder) whose path starts with '/public/' are always allowed. They,
    as well as OPTIONS requests, can be made without a bearer token.
    Unless [KERBEROS] is used (see section 10 below), all other requests
    SHOULD present a bearer token with sufficient access scope, using a
    header of the following form (no double quotes here):

       Authorization: Bearer <access_token>

    In addition, providing the access token via a HTTP query parameter
    for GET requests MAY be supported by the server, although its use
    is not recommended, due to its security deficiencies; see [BEARER,
    section 2.3].


de Jong                                                         [Page 9]

         
Internet-Draft              remoteStorage                  December 2014



10. Application-first bearer token issuance

    To make a remoteStorage server available as 'the remoteStorage of
    <account> at <host>', exactly one link of the following format
    SHOULD be added to the WebFinger record [WEBFINGER] of <account> at
    <host>:

    {
      "href": <storage_root>,
      "rel": "remotestorage",
      "properties": {
        "http://remotestorage.io/spec/version": <storage_api>,
        "http://tools.ietf.org/html/rfc6749#section-4.2": <auth-dialog>,
        ... : ... ,
      }
    }

    Here <storage_root> and <storage_api> are as per "Session
    description" above, and <auth-dialog> SHOULD be either null or a
    URL where an OAuth 2.0 implicit-grant flow dialog [OAUTH] is
    presented.

    If <auth-dialog> is a URL, the user can supply their credentials
    for accessing the account (how, is out of scope), and allow or
    reject a request by the connecting application to obtain a bearer
    token for a certain list of access scopes. Note that an account
    will often belong to just one human user, but may also belong to a
    group of multiple users (the remoteStorage of <group> at <host>).

    If <auth-dialog> is null, the client will not have a way to obtain
    an access token, and SHOULD send all requests without Authorization
    header, and rely on Kerberos [KERBEROS] instead for requests that
    would normally be sent with a bearer token, but servers SHOULD NOT
    impose any such access barriers for resources that would normally
    not require an access token.

    The '...' ellipses indicate that more properties may be present.
    Non-breaking examples that have been proposed so far, include a
    "http://tools.ietf.org/html/rfc6750#section-2.3" property, set to
    the string value "true" if the server supports passing the bearer
    token in the URI query parameter as per section 2.3 of [BEARER],
    instead of in the request header.


de Jong                                                        [Page 10]

         
Internet-Draft              remoteStorage                  December 2014



    Another example is "http://tools.ietf.org/html/rfc7233" with a
    string value of "GET" if Content-Range headers are supported for
    GET requests as per [RANGE], "PUT" if they are supported for PUT
    requests, and "GET,PUT" if supported for both.

    Both these proposals are non-breaking extensions, since the client
    will have a way to work around it if these features are not present
    (e.g. retrieve the protected resource asynchronously in the first
    case, or request the entire resource in the second case).

    A "http://remotestorage.io/spec/web-authoring" property has been
    proposed with a string value of the fully qualified domain name to
    which web authoring content is published if the server supports web
    authoring as per [AUTHORING]. Note that this extension is a breaking
    extension in the sense that it divides users into "haves", whose
    remoteStorage accounts allow them to author web content, and
    "have-nots", whose remoteStorage account does not support this
    functionality.

    The server MAY expire bearer tokens, and MAY require the user to
    register applications as OAuth clients before first use; if no
    client registration is required, then the server MAY ignore the
    client_id parameter in favor of relying on the redirect_uri
    parameter for client identification.

11. Storage-first bearer token issuance

    The provider MAY also present a dashboard to the user, where they
    have some way to add open web app manifests [MANIFEST]. Adding a
    manifest to the dashboard is considered equivalent to clicking
    'accept' in the dialog of the application-first flow. Removing one
    is considered equivalent to revoking its access token.

    As an equivalent to OAuth's 'scope' parameter, a 'datastores-access'
    field SHOULD be present in the root of such an application manifest
    document, with entries <module> -> '{"access": "readonly"}' for
    <level> 'r' or '{"access": "readwrite"}' for <level> 'rw', as
    prescribed in [DATASTORE].

    When the user gestures they want to use a certain application whose
    manifest is present on the dashboard, the dashboard SHOULD redirect
    to the application or open it in a new window. To mimic coming back


de Jong                                                        [Page 11]

         
Internet-Draft              remoteStorage                  December 2014


    from the OAuth dialog, it MAY add 'access_token' and 'scope'
    fields to the URL fragment.

    Regardless of whether 'access_token' and 'scope' are specified, it
    SHOULD add a 'remotestorage' field to the URL fragment, with a
    value of the form <account> '@' <host>. When the application detects
    this parameter, it SHOULD resolve the WebFinger record for <account>
    at <host> and extract the <storage_root> and <storage_api>
    information.

    If no access_token was given, then the application SHOULD also
    extract the <auth_endpoint> information from WebFinger, and continue
    as per application-first bearer token issuance.

    Note that whereas a remoteStorage server SHOULD offer support for
    the application-first flow with WebFinger and OAuth, it MAY choose
    not to support the storage-first flow, provided that users will
    easily remember their <account> '@' <host> WebFinger address at that
    provider. Applications SHOULD, however, support both flows, which
    means checking the URL for a 'remotestorage' parameter, but giving
    the user a way to specify the WebFinger address if there is none.

    If a server provides an application manifest dashboard, then it
    SHOULD merge the list of applications there with the list of
    issued access tokens as specified by OAuth into one list. Also,
    the interface for revoking an access token as specified by OAuth
    SHOULD coincide with removing an application from the dashboard.

    Servers MAY also provide a way to create access tokens directly from
    their user interface. Such functionality would be aimed mainly at
    developers, to manually copy and paste a token into a script or
    debug tool, thus bypassing the need for an OAuth dance. Clients
    SHOULD NOT rely on this in production.

12. Example wire transcripts

    The following examples are not normative ("\" indicates a line was
    wrapped).

12.1. WebFinger

    In application-first, an in-browser application might issue the
    following request, using XMLHttpRequest and CORS:


de Jong                                                        [Page 12]

         
Internet-Draft              remoteStorage                  December 2014



         GET /.well-known/webfinger?resource=acct:michiel@michielbdejon\
g.com HTTP/1.1
         Host: michielbdejong.com

    and the server's response might look like this:

         HTTP/1.1 200 OK
         Access-Control-Allow-Origin: *
         Access-Control-Allow-Methods: GET
         Access-Control-Allow-Headers: If-Match, If-None-Match
         Access-Control-Expose-Headers: ETag, Content-Length
         Content-Type: application/jrd+json

         {
           "links":[{
             "href": "https://michielbdejong.com:7678/inbox",
             "rel": "post-me-anything"
           }, {
             "href": "https://michielbdejong.com/me.jpg",
             "rel": "avatar"
           }, {
             "href": "https://3pp.io:4439/storage/michiel",
             "rel": "remotestorage",
             "properties": {
               "http://remotestorage.io/spec/version": "draft-dejong-re\
motestorage-04",
               "http://tools.ietf.org/html/rfc6749#section-4.2": "https\
://3pp.io:4439/oauth/michiel",
               "http://tools.ietf.org/html/rfc6750#section-2.3": false,
               "http://tools.ietf.org/html/rfc7233": false,
               "http://remotestorage.io/spec/web-authoring": false
             }
           }]
         }

12.2. OAuth dialog form

    Once the in-browser application has discovered the server's OAuth
    end-point, it will typically redirect the user to this URL, in
    order to obtain a bearer token. Say the application is hosted on
    https://drinks-unhosted.5apps.com/ and wants read-write access to
    the account's "myfavoritedrinks" scope:


de Jong                                                        [Page 13]

         
Internet-Draft              remoteStorage                  December 2014



        GET /oauth/michiel?redirect_uri=https%3A%2F%2Fdrinks-unhosted.5\
apps.com%2F&scope=myfavoritedrinks%3Arw&client_id=https%3A%2F%2Fdrinks-\
unhosted.5apps.com&response_type=token HTTP/1.1
        Host: 3pp.io

    The server's response might look like this (truncated for brevity):

        HTTP/1.1 200 OK

        <!DOCTYPE html>
        <html lang="en">
          <head>
            <title>Allow access?</title>
        ...

12.3. OAuth dialog form submission

    When the user submits the form, the request would look something
    like this:

        POST /oauth HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://3pp.io:4439
        Content-Type: application/x-www-form-urlencoded
        Referer: https://3pp.io:4439/oauth/michiel?redirect_uri=https%3\
A%2F%2Fdrinks-unhosted.5apps.com%2F&scope=myfavoritedrinks%3Arw&client_\
id=https%3A%2F%2Fdrinks-unhosted.5apps.com&response_type=token

        client_id=https%3A%2F%2Fdrinks-unhosted.5apps.com&redirect_uri=\
https%3A%2F%2Fdrinks-unhosted.5apps.com%2F&response_type=token&scope=my\
favoritedrinks%3Arw&state=&username=michiel&password=something&allow=Al\
low

    To which the server could respond with a 302 redirect, back to the
    origin of the requesting application:

        HTTP/1.1 302 Found
        Location:https://drinks-unhosted.5apps.com/#access_token=j2YnGt\
XjzzzHNjkd1CJxoQubA1o%3D&token_type=bearer&state=

12.4. OPTIONS preflight



de Jong                                                        [Page 14]

         
Internet-Draft              remoteStorage                  December 2014


    When an in-browser application makes a cross-origin request which
    may affect the server-state, the browser will make a preflight
    request first, with the OPTIONS verb, for instance:

        OPTIONS /storage/michiel/myfavoritedrinks/ HTTP/1.1
        Host: 3pp.io:4439
        Access-Control-Request-Method: GET
        Origin: https://drinks-unhosted.5apps.com
        Access-Control-Request-Headers: Authorization
        Referer: https://drinks-unhosted.5apps.com/

    To which the server can for instance respond:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Access-Control-Allow-Methods: GET, PUT, DELETE
        Access-Control-Allow-Headers: Authorization, Content-Length, Co\
ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match

12.5. Initial PUT

    An initial PUT may contain an 'If-None-Match: *' header, like this:

        PUT /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Content-Length: 91
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-None-Match: *

        {"name":"test","@context":"http://remotestorage.io/spec/modules\
/myfavoritedrinks/drink"}

    And the server may respond with either a 201 Created or a 200 OK
    status:

        HTTP/1.1 201 Created
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694045000"

12.6. Subsequent PUT


de Jong                                                        [Page 15]

         
Internet-Draft              remoteStorage                  December 2014



    A subsequent PUT may contain an 'If-Match' header referring to the
    ETag previously returned, like this:

        PUT /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Content-Length: 91
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-Match: "1382694045000"

        {"name":"test", "updated":true, "@context":"http://remotestorag\
e.io/spec/modules/myfavoritedrinks/drink"}

    And the server may respond with a 412 Conflict or a 200 OK status:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

12.7. GET

    A GET request would also include the bearer token, and optionally
    an If-None-Match header:

        GET /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Referer: https://drinks-unhosted.5apps.com/?
        If-None-Match: "1382694045000", "1382694048000"

    And the server may respond with a 304 Not Modified status:

        HTTP/1.1 304 Not Modified
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

    Or a 200 OK status, plus a response body:

        HTTP/1.1 200 OK


de Jong                                                        [Page 16]

         
Internet-Draft              remoteStorage                  December 2014


        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Content-Type: application/json; charset=UTF-8
        Content-Length: 106
        ETag: "1382694048000"
        Expires: 0

        {"name":"test", "updated":true, "@context":"http://remotestora\
ge.io/spec/modules/myfavoritedrinks/drink"}

    If the GET URL would have been "/storage/michiel/myfavoritedrinks/",
    a 200 OK response would have a folder description as the response
    body:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Content-Type: application/ld+json
        Content-Length: 171
        ETag: "1382694048000"
        Expires: 0

        {"@context":"http://remotestorage.io/spec/folder-version","ite\
ms":{"test":{"ETag":"1382694048000","Content-Type":"application/json; \
charset=UTF-8","Content-Length":106}}}

    If the GET URL would have been a non-existing document like
    "/storage/michiel/myfavoritedrinks/x", the response would have a 404
    Not Found status, and no ETag header:

        HTTP/1.1 404 Not Found
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com

12.8. DELETE

    A DELETE request may look like this:

        DELETE /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-Match: "1382694045000"



de Jong                                                        [Page 17]

         
Internet-Draft              remoteStorage                  December 2014


    And the server may respond with a 412 Conflict or a 200 OK status:

        HTTP/1.1 412 Conflict
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

13. Distributed versioning

    This section is non-normative, and is intended to explain some of
    the design choices concerning ETags and folder listings. At the
    same time it will hopefully help readers who intend to develop an
    application that uses remoteStorage as its per-user data storage.
    When multiple clients have read/write access to the same document,
    versioning conflicts may occur. For instance, client A may make
    a PUT request that changes the document from version 1 to version
    2, after which client B may make a PUT request attempting to change
    the same document from version 1 to version 3.

    In this case, client B can add an 'If-Match: "1"' header, which
    would trigger a 412 Conflict response code, since the current
    version ("2") does not match the version required as a condition by
    the header If-Match header ("1").

    Client B is now aware of the conflict, and may consult the user,
    saying the update to version 3 failed. The user may then choose,
    through the user interface of client B, whether version 2 or
    version 3 should be kept, or maybe the document should be reverted
    on the server to version 1, or a merged version 4 is needed. Client
    B may then make a request that puts the document to the version the
    user wishes; this time setting an 'If-Match: "2"' header instead.

    Both client A and client B would periodically poll the root
    folder of each scope they have access to, to see if the version
    of the root folder changed. If it did, then one of the versions
    listed in there will necessarily have changed, and the client can
    make a GET request to that child folder or document, to obtain
    its latest version.

    Because an update in a document will result in a version change of
    its containing folder, and that change will propagate all the way
    to the root folder, it is not necessary to poll each document for
    changes individually.



de Jong                                                        [Page 18]

         
Internet-Draft              remoteStorage                  December 2014


    As an example, the root folder may contain 10 directories,
    each of which contain 10 directories, which each contain 10
    documents, so their paths would be for instance '/0/0/1', '/0/0/2',
    etcetera. Then one GET request to the root folder '/' will be
    enough to know if any of these 1000 documents has changed.

    Say document '/7/9/2' has changed; then the GET request to '/' will
    come back with a different ETag, and entry '7/' will have a
    different value in its JSON content. The client could then request
    '/7/', '/7/9/', and '/7/9/2' to narrow down the one document that
    caused the root folder's ETag to change.

    Note that the remoteStorage server does not get involved in the
    conflict resolution. It keeps the canonical current version at all
    times, and allows clients to make conditional GET and PUT requests,
    but it is up to whichever client discovers a given version
    conflict, to resolve it.

14. Security Considerations

    To prevent man-in-the-middle attacks, the use of https instead of
    http is important for both the interface itself and all end-points
    involved in WebFinger, OAuth, and (if present) the storage-first
    application launch dashboard.

    A malicious party could link to an application, but specifying a
    remoteStorage account address that it controls, thus tricking the
    user into using a trusted application to send sensitive data to the
    wrong remoteStorage server. To mitigate this, applications SHOULD
    clearly display to which remoteStorage server they are sending the
    user's data.

    Applications could request scopes that the user did not intend to
    give access to. The user SHOULD always be prompted to carefully
    review which scopes an application is requesting.

    An application may upload malicious html pages and then trick the
    user into visiting them, or upload malicious client-side scripts,
    that take advantage of being hosted on the user's domain name. The
    origin on which the remoteStorage server has its interface SHOULD
    therefore NOT be used for anything else, and the user SHOULD be
    warned not to visit any web pages on that origin. In particular, the
    OAuth dialog and launch dashboard or token revokation interface


de Jong                                                        [Page 19]

         
Internet-Draft              remoteStorage                  December 2014


    SHOULD be on a different origin than the remoteStorage interface.

    Where the use of bearer tokens is impractical, a user may choose to
    store documents on hard-to-guess URLs whose path after
    <storage_root> starts with '/public/', while sharing this URL only
    with the intended audience. That way, only parties who know the
    document's hard-to-guess URL, can access it. The server SHOULD
    therefore make an effort to detect and stop brute-force attacks that
    attempt to guess the location of such documents.

    The server SHOULD also detect and stop denial-of-service attacks
    that aim to overwhelm its interface with too much traffic.

15. IANA Considerations

    This document registers the 'remotestorage' link relation, as well
    as the following WebFinger properties:
      * "http://remotestorage.io/spec/version"
      * "http://tools.ietf.org/html/rfc6749#section-4.2"
      * "http://tools.ietf.org/html/rfc6750#section-2.3"
      * "http://tools.ietf.org/html/rfc7233"
      * "http://remotestorage.io/spec/web-authoring"

16. Acknowledgements

    The authors would like to thank everybody who contributed to the
    development of this protocol, including Kenny Bentley, Javier Diaz,
    Daniel Groeber, Bjarni Runar, Jan Wildeboer, Charles Schultz, Peter
    Svensson, Valer Mischenko, Michiel Leenaars, Jan-Christoph
    Borchardt, Garret Alfert, Sebastian Kippe, Max Wiehle, Melvin
    Carvalho, Martin Stadler, Geoffroy Couprie, Niklas Cathor, Marco
    Stahl, James Coglan, Ken Eucker, Daniel Brolund, elf Pavlik, Nick
    Jennings, Markus Sabadello, Steven te Brinke, Matthias Treydte,
    Rick van Rein, Mark Nottingham, Julian Reschke, and Markus
    Lanthaler, among many others.

17. References

17.1. Normative References

    [WORDS]
        Bradner, S., "Key words for use in RFCs to Indicate Requirement
        Levels", BCP 14, RFC 2119, March 1997.


de Jong                                                        [Page 20]

         
Internet-Draft              remoteStorage                  December 2014



    [IRI]
        Duerst, M., "Internationalized Resource Identifiers (IRIs)",
        RFC 3987, January 2005.

    [WEBFINGER]
        Jones, P., Salguerio, G., Jones, M, and Smarr, J.,
        "WebFinger", RFC7033, September 2013.

    [OAUTH]
        "Section 4.2: Implicit Grant", in: Hardt, D. (ed), "The OAuth
        2.0 Authorization Framework", RFC6749, October 2012.

17.2. Informative References

    [HTTPS]
        Rescorla, E., "HTTP Over TLS", RFC2818, May 2000.

    [HTTP]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Semantics and Content", RFC7231, June 2014.

    [COND]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Conditional Requests", RFC7232, June 2014.

    [RANGE]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Conditional Requests", RFC7233, June 2014.

    [SPDY]
        Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", http://
        www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1,
        September 2013.

    [JSON-LD]
        M. Sporny, G. Kellogg, M. Lanthaler, "JSON-LD 1.0", W3C
        Proposed Recommendation,
        http://www.w3.org/TR/2014/REC-json-ld-20140116/, January 2014.

    [CORS]
        van Kesteren, Anne (ed), "Cross-Origin Resource Sharing --
        W3C Candidate Recommendation 29 January 2013",


de Jong                                                        [Page 21]

         
Internet-Draft              remoteStorage                  December 2014


        http://www.w3.org/TR/cors/, January 2013.

    [MANIFEST]
        Mozilla Developer Network (ed), "App manifest -- Revision
        330541", https://developer.mozilla.org/en-
        US/Apps/Build/Manifest$revision/566677, April 2014.

    [DATASTORE]
        "WebAPI/DataStore", MozillaWiki, retrieved May 2014.
        https://wiki.mozilla.org/WebAPI/DataStore#Manifest

    [KERBEROS]
        C. Neuman et al., "The Kerberos Network Authentication Service
        (V5)", RFC4120, July 2005.

    [BEARER]
        M. Jones, D. Hardt, "The OAuth 2.0 Authorization Framework:
        Bearer Token Usage", RFC6750, October 2012.

    [AUTHORING]
        "Using remoteStorage for web authoring", reSite wiki, retrieved
        September 2014. https://github.com/michielbdejong/resite/wiki
        /Using-remoteStorage-for-web-authoring

18. Authors' addresses

    Michiel B. de Jong
    IndieHosters

    Email: michiel@michielbdejong.com


    F. Kooman
    (independent)

    Email: fkooman@tuxed.net









de Jong                                                        [Page 22]

        Html markup produced by rfcmarkup 1.111, available from https://tools.ietf.org/tools/rfcmarkup/
        ================================================ FILE: test/test-pages/ietf-1/source.html ================================================ draft-dejong-remotestorage-04 - remoteStorage
        [Docs] [txt|pdf] [Tracker] [Email] [Diff1] [Diff2] [Nits]

        Versions: 00 01 02 03 04 

        INTERNET DRAFT                                      Michiel B. de Jong
Document: draft-dejong-remotestorage-04                   IndieHosters
                                                             F. Kooman
Intended Status: Proposed Standard                       (independent)
Expires: 18 June 2015                                 15 December 2014


                            remoteStorage

Abstract

    This draft describes a protocol by which client-side applications,
    running inside a web browser, can communicate with a data storage
    server that is hosted on a different domain name. This way, the
    provider of a web application need not also play the role of data
    storage provider. The protocol supports storing, retrieving, and
    removing individual documents, as well as listing the contents of an
    individual folder, and access control is based on bearer tokens.

Status of this Memo

   This Internet-Draft is submitted in full conformance with the
   provisions of BCP 78 and BCP 79.

   Internet-Drafts are working documents of the Internet Engineering
   Task Force (IETF).  Note that other groups may also distribute
   working documents as Internet-Drafts.  The list of current Internet-
   Drafts is at http://datatracker.ietf.org/drafts/current/.

   Internet-Drafts are draft documents valid for a maximum of six months
   and may be updated, replaced, or obsoleted by other documents at any
   time.  It is inappropriate to use Internet-Drafts as reference
   material or to cite them other than as "work in progress."

   This Internet-Draft will expire on 15 December 2014.

Copyright Notice

   Copyright (c) 2014 IETF Trust and the persons identified as the
   document authors. All rights reserved.

   This document is subject to BCP 78 and the IETF Trust's Legal
   Provisions Relating to IETF Documents
   (http://trustee.ietf.org/license-info) in effect on the date of
   publication of this document.  Please review these documents
   carefully, as they describe your rights and restrictions with respect
   to this document.  Code Components extracted from this document must
   include Simplified BSD License text as described in Section 4.e of
   the Trust Legal Provisions and are provided without warranty as
   described in the Simplified BSD License.


de Jong                                                         [Page 1]

        
Internet-Draft              remoteStorage                  December 2014


Table of Contents

   1. Introduction...................................................2
   2. Terminology....................................................3
   3. Storage model..................................................3
   4. Requests.......................................................4
   5. Response codes.................................................7
   6. Versioning.....................................................7
   7. CORS headers...................................................8
   8. Session description............................................8
   9. Bearer tokens and access control...............................9
  10. Application-first bearer token issuance.......................10
  11. Storage-first bearer token issuance...........................11
  12. Example wire transcripts......................................12
     12.1. WebFinger................................................12
     12.2. OAuth dialog form........................................13
     12.3. OAuth dialog form submission.............................14
     12.4. OPTIONS preflight........................................15
     12.5. Initial PUT..............................................15
     12.6. Subsequent PUT...........................................16
     12.7. GET......................................................16
     12.8. DELETE...................................................17
  13. Distributed versioning........................................17
  14. Security Considerations.......................................19
  15. IANA Considerations...........................................20
  16. Acknowledgments...............................................20
  17. References....................................................21
     17.1. Normative References.....................................21
     17.2. Informative References...................................21
  18. Authors' addresses............................................22


1.  Introduction

    Many services for data storage are available over the internet. This
    specification describes a vendor-independent interface for such
    services. It is based on https, CORS and bearer tokens. The
    metaphor for addressing data on the storage is that of folders
    containing documents and subfolders. The actions the interface
    exposes are:

       *  GET a folder: retrieve the names and current versions of the
          documents and subfolders currently contained by the folder


de Jong                                                         [Page 2]

        
Internet-Draft              remoteStorage                  December 2014


       *  GET a document: retrieve its content type, current version,
          and contents

       *  PUT a document: store a new version, its content type, and
          contents, conditional on the current version

       *  DELETE a document: remove it from the storage, conditional on
          the current version

       *  HEAD a folder or document: like GET, but omitting the response
          body

    The exact details of these four actions are described in this
    specification.

2. Terminology

    The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
    "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
    document are to be interpreted as described in RFC 2119 [WORDS].

    "SHOULD" and "SHOULD NOT" are appropriate when valid exceptions to a
    general requirement are known to exist or appear to exist, and it is
    infeasible or impractical to enumerate all of them.  However, they
    should not be interpreted as permitting implementors to fail to
    implement the general requirement when such failure would result in
    interoperability failure.

3. Storage model

    The server stores data in nodes that form a tree structure.
    Internal nodes are called 'folders' and leaf nodes are called
    'documents'. For a folder, the server stores references to nodes
    contained in the folder, and it should be able to produce a list of
    them, with for each contained item:

       * item name
       * item type (folder or document)
       * current version
       * content type
       * content length

    For a document, the server stores, and should be able to produce:


de Jong                                                         [Page 3]

        
Internet-Draft              remoteStorage                  December 2014



       * current version
       * content type
       * content length
       * content

4. Requests

    Client-to-server requests SHOULD be made over https [HTTPS], and
    servers MUST comply with HTTP/1.1 [HTTP]. Specifically, they
    MUST support chunked transfer coding on PUT requests. Servers MAY
    also offer an optional switch from https to SPDY [SPDY].

    A request is considered successful if the HTTP response code is in
    the 2xx range (e.g. 200 OK, 201 Created), and unsuccessful if an
    error occurred or a condition was not met (response code e.g. 404
    Not Found, 304 Not Modified).

    The root folder of the storage tree is represented by the following
    URL:

        URI_ENCODE( <storage_root> '/' )

    Subsequently, if <parent_folder> is the URL of a folder, then the
    URL of an item contained in it is:

        URI_ENCODE( <parent_folder> <document_name> )

    for a document, or:

        URI_ENCODE( <parent_folder> <folder_name> '/' )

    for a folder. Item names MAY contain all characters except '/' and
    the null character, and MUST NOT have zero length.

    A document description is a map containing one string-valued 'ETag'
    field, one string-valued 'Content-Type' and one integer-valued
    'Content-Length' field. They represent the document's current
    version, its content type, and its content length respectively. Note
    that content length is measured in octets (bytes), not in
    characters.

    A folder description is a map containing a string-valued 'ETag'


de Jong                                                         [Page 4]

        
Internet-Draft              remoteStorage                  December 2014


    field, representing the folder's current version.

    A successful GET request to a folder MUST be responded to with a
    JSON-LD [JSON-LD] document (content type 'application/ld+json'),
    containing as its 'items' field a map in which contained documents
    appear as entries <item_name> to a document description, and
    contained non-empty folders appear as entries <item_name> '/' to a
    folder description. It MUST also contain an '@context' field with
    the value 'http://remotestorage.io/spec/folder-description'. For
    instance:

       {
         "@context": "http://remotestorage.io/spec/folder-description",
         "items": {
           "abc": {
             "ETag": "DEADBEEFDEADBEEFDEADBEEF",
             "Content-Type": "image/jpeg",
             "Content-Length": 82352
           },
           "def/": {
             "ETag": "1337ABCD1337ABCD1337ABCD"
           }
         }
       }

    All folders are treated as existing, and therefore GET requests to
    untouched folders SHOULD be responded to with a folder description
    with no items (the items field set to '{}'). However, an empty
    folder MUST NOT be listed as an item in its parent folder.

    Also, since folders exist automatically, PUT and DELETE requests
    only need to be made to documents, and never to folders. A document
    PUT will make all ancestor folders along its path become non-empty;
    deleting the last document from a subtree will make that whole
    subtree become empty. Folders will therefore show up in their parent
    folder descriptions if and only if their subtree contains at least
    one document.

    A successful GET request to a document SHOULD be responded to with
    the full document contents in the body, the document's content type
    in a 'Content-Type' header, its content length in octets (not in
    characters) in a 'Content-Length' header, and the document's current
    version as a strong ETag in an 'ETag' header.


de Jong                                                         [Page 5]

        
Internet-Draft              remoteStorage                  December 2014



    Note that the use of strong ETags prohibits changing the response
    body based on request headers; in particular, the server will not be
    able to serve the same document uncompressed to some clients and
    gzipped when requested by the client, since the two bodies would not
    be identical byte-for-byte.

    Servers MAY support Content-Range headers [RANGE] on GET requests,
    but whether or not they do SHOULD be announced through the <ranges>
    variable mentioned below in section 10.

    A successful PUT request to a document MUST result in:

       * the request body being stored as the document's new content,
       * parent and further ancestor folders being silently created as
         necessary, with the document (name and version) being added to
         its parent folder, and each folder added to its subsequent
         parent,
       * the value of its Content-Type header being stored as the
         document's new content type,
       * its version being updated, as well as that of its parent folder
         and further ancestor folders, using a strong validator [HTTP,
         section 7.2].

    The response MUST contain a strong ETag header, with the document's
    new version (for instance a hash of its contents) as its value.

    A successful DELETE request to a document MUST result in:

       * the deletion of that document from the storage, and from its
         parent folder,
       * silent deletion of the parent folder if it is left empty by
         this, and so on for further ancestor folders,
       * the version of its parent folder being updated, as well as that
         of further ancestor folders.

    A successful OPTIONS request SHOULD be responded to as described in
    the CORS section below.

    A successful HEAD request SHOULD be responded to like to the
    equivalent GET request, but omitting the response body.




de Jong                                                         [Page 6]

        
Internet-Draft              remoteStorage                  December 2014


5. Response codes

    Response codes SHOULD be given as defined by [HTTP, section 6] and
    [BEARER, section 3.1]. The following is a non-normative checklist
    of status codes that are likely to occur in practice:

       * 500 if an internal server error occurs,
       * 429 if the client makes too frequent requests or is suspected
             of malicious activity,
       * 414 if the request URI is too long,
       * 416 if Range requests are supported by the server and the Range
             request can not be satisfied,
       * 401 for all requests that don't have a bearer token with
             sufficient permissions,
       * 404 for all DELETE and GET requests to documents that do not
             exist on the storage,
       * 304 for a conditional GET request whose pre-condition
             fails (see "Versioning" below),
       * 409 for a PUT request where any folder name in the path
             clashes with an existing document's name at the same
             level, or where the document name coincides with an
             existing folder's name at the same level.
       * 412 for a conditional PUT or DELETE request whose pre-condition
             fails (see "Versioning" below),
       * 507 in case the account is over its storage quota,
       * 4xx for all malformed requests (e.g. foreign characters in the
             path), as well as for all PUT and DELETE requests to
             folders,
       * 2xx for all successful requests.

    Clients SHOULD also handle the case where a response takes too long
    to arrive, or where no response is received at all.

6. Versioning

    All successful requests MUST return an 'ETag' header [HTTP] with, in
    the case of GET, the current version, in the case of PUT, the new
    version, and in case of DELETE, the version that was deleted. All
    successful GET requests MUST return an 'Expires: 0' header. PUT and
    DELETE requests MAY have an 'If-Match' request header [COND], and
    MUST fail with a 412 response code if that doesn't match the
    document's current version.



de Jong                                                         [Page 7]

        
Internet-Draft              remoteStorage                  December 2014


    GET requests MAY have a comma-separated list of revisions in an
    'If-None-Match' header [COND], and SHOULD be responded to with a 304
    response if that list includes the document or folder's current
    version. A PUT request MAY have an 'If-None-Match: *' header [COND],
    in which case it MUST fail with a 412 response code if the document
    already exists.

    In all 'ETag', 'If-Match' and 'If-None-Match' headers, revision
    strings should appear inside double quotes (").

    A provider MAY offer version rollback functionality to its users,
    but this specification does not define the user interface for that.

7. CORS headers

    All responses MUST carry CORS headers [CORS]. The server MUST also
    reply to OPTIONS requests as per CORS. For GET requests, a wildcard
    origin MAY be returned, but for PUT and DELETE requests, the
    response MUST echo back the Origin header sent by the client.

8. Session description

    The information that a client needs to receive in order to be able
    to connect to a server SHOULD reach the client as described in the
    'bearer token issuance' sections below. It consists of:

       * <storage_root>, consisting of 'https://' followed by a server
         host, and optionally a server port and a path prefix as per
         [IRI]. Examples:
         * 'https://example.com' (host only)
         * 'https://example.com:8080' (host and port)
         * 'https://example.com/path/to/storage' (host, port and
           path prefix; note there is no trailing slash)
       * <access_token> as per [OAUTH]. The token SHOULD be hard to
         guess and SHOULD NOT be reused from one client to another. It
         can however be reused in subsequent interactions with the same
         client, as long as that client is still trusted. Example:
         * 'ofb24f1ac3973e70j6vts19qr9v2eei'
       * <storage_api>, always 'draft-dejong-remotestorage-04' for this
         alternative version of the specification.

    The client can make its requests using https with CORS and bearer
    tokens, to the URL that is the concatenation of <storage_root> with


de Jong                                                         [Page 8]

        
Internet-Draft              remoteStorage                  December 2014


    '/' plus one or more <folder> '/' strings indicating a path in the
    folder tree, followed by zero or one <document> strings, indicating
    a document. For example, if <storage_root> is
    "https://storage.example.com/bob", then to retrieve the folder
    contents of the /public/documents/ folder, or to retrieve a
    'draft.txt' document from that folder, the client would make
    requests to, respectively:

    * https://storage.example.com/bob/public/documents/
    * https://storage.example.com/bob/public/documents/draft.txt

9. Bearer tokens and access control

    A bearer token represents one or more access scopes. These access
    scopes are represented as strings of the form <module> <level>,
    where the <module> string SHOULD be lower-case alphanumerical, other
    than the reserved word 'public', and <level> can be ':r' or ':rw'.
    The access the bearer token gives is the sum of its access scopes,
    with each access scope representing the following permissions:

    '*:rw') any request,

    '*:r') any GET or HEAD request,

    <module> ':rw') any requests to paths that start with
                    '/' <module> '/' or '/public/' <module> '/',

    <module> ':r') any GET or HEAD requests to paths that start with
                   '/' <module> '/' or '/public/' <module> '/',

    As a special exceptions, GET requests to a document (but not a
    folder) whose path starts with '/public/' are always allowed. They,
    as well as OPTIONS requests, can be made without a bearer token.
    Unless [KERBEROS] is used (see section 10 below), all other requests
    SHOULD present a bearer token with sufficient access scope, using a
    header of the following form (no double quotes here):

       Authorization: Bearer <access_token>

    In addition, providing the access token via a HTTP query parameter
    for GET requests MAY be supported by the server, although its use
    is not recommended, due to its security deficiencies; see [BEARER,
    section 2.3].


de Jong                                                         [Page 9]

        
Internet-Draft              remoteStorage                  December 2014



10. Application-first bearer token issuance

    To make a remoteStorage server available as 'the remoteStorage of
    <account> at <host>', exactly one link of the following format
    SHOULD be added to the WebFinger record [WEBFINGER] of <account> at
    <host>:

    {
      "href": <storage_root>,
      "rel": "remotestorage",
      "properties": {
        "http://remotestorage.io/spec/version": <storage_api>,
        "http://tools.ietf.org/html/rfc6749#section-4.2": <auth-dialog>,
        ... : ... ,
      }
    }

    Here <storage_root> and <storage_api> are as per "Session
    description" above, and <auth-dialog> SHOULD be either null or a
    URL where an OAuth 2.0 implicit-grant flow dialog [OAUTH] is
    presented.

    If <auth-dialog> is a URL, the user can supply their credentials
    for accessing the account (how, is out of scope), and allow or
    reject a request by the connecting application to obtain a bearer
    token for a certain list of access scopes. Note that an account
    will often belong to just one human user, but may also belong to a
    group of multiple users (the remoteStorage of <group> at <host>).

    If <auth-dialog> is null, the client will not have a way to obtain
    an access token, and SHOULD send all requests without Authorization
    header, and rely on Kerberos [KERBEROS] instead for requests that
    would normally be sent with a bearer token, but servers SHOULD NOT
    impose any such access barriers for resources that would normally
    not require an access token.

    The '...' ellipses indicate that more properties may be present.
    Non-breaking examples that have been proposed so far, include a
    "http://tools.ietf.org/html/rfc6750#section-2.3" property, set to
    the string value "true" if the server supports passing the bearer
    token in the URI query parameter as per section 2.3 of [BEARER],
    instead of in the request header.


de Jong                                                        [Page 10]

        
Internet-Draft              remoteStorage                  December 2014



    Another example is "http://tools.ietf.org/html/rfc7233" with a
    string value of "GET" if Content-Range headers are supported for
    GET requests as per [RANGE], "PUT" if they are supported for PUT
    requests, and "GET,PUT" if supported for both.

    Both these proposals are non-breaking extensions, since the client
    will have a way to work around it if these features are not present
    (e.g. retrieve the protected resource asynchronously in the first
    case, or request the entire resource in the second case).

    A "http://remotestorage.io/spec/web-authoring" property has been
    proposed with a string value of the fully qualified domain name to
    which web authoring content is published if the server supports web
    authoring as per [AUTHORING]. Note that this extension is a breaking
    extension in the sense that it divides users into "haves", whose
    remoteStorage accounts allow them to author web content, and
    "have-nots", whose remoteStorage account does not support this
    functionality.

    The server MAY expire bearer tokens, and MAY require the user to
    register applications as OAuth clients before first use; if no
    client registration is required, then the server MAY ignore the
    client_id parameter in favor of relying on the redirect_uri
    parameter for client identification.

11. Storage-first bearer token issuance

    The provider MAY also present a dashboard to the user, where they
    have some way to add open web app manifests [MANIFEST]. Adding a
    manifest to the dashboard is considered equivalent to clicking
    'accept' in the dialog of the application-first flow. Removing one
    is considered equivalent to revoking its access token.

    As an equivalent to OAuth's 'scope' parameter, a 'datastores-access'
    field SHOULD be present in the root of such an application manifest
    document, with entries <module> -> '{"access": "readonly"}' for
    <level> 'r' or '{"access": "readwrite"}' for <level> 'rw', as
    prescribed in [DATASTORE].

    When the user gestures they want to use a certain application whose
    manifest is present on the dashboard, the dashboard SHOULD redirect
    to the application or open it in a new window. To mimic coming back


de Jong                                                        [Page 11]

        
Internet-Draft              remoteStorage                  December 2014


    from the OAuth dialog, it MAY add 'access_token' and 'scope'
    fields to the URL fragment.

    Regardless of whether 'access_token' and 'scope' are specified, it
    SHOULD add a 'remotestorage' field to the URL fragment, with a
    value of the form <account> '@' <host>. When the application detects
    this parameter, it SHOULD resolve the WebFinger record for <account>
    at <host> and extract the <storage_root> and <storage_api>
    information.

    If no access_token was given, then the application SHOULD also
    extract the <auth_endpoint> information from WebFinger, and continue
    as per application-first bearer token issuance.

    Note that whereas a remoteStorage server SHOULD offer support for
    the application-first flow with WebFinger and OAuth, it MAY choose
    not to support the storage-first flow, provided that users will
    easily remember their <account> '@' <host> WebFinger address at that
    provider. Applications SHOULD, however, support both flows, which
    means checking the URL for a 'remotestorage' parameter, but giving
    the user a way to specify the WebFinger address if there is none.

    If a server provides an application manifest dashboard, then it
    SHOULD merge the list of applications there with the list of
    issued access tokens as specified by OAuth into one list. Also,
    the interface for revoking an access token as specified by OAuth
    SHOULD coincide with removing an application from the dashboard.

    Servers MAY also provide a way to create access tokens directly from
    their user interface. Such functionality would be aimed mainly at
    developers, to manually copy and paste a token into a script or
    debug tool, thus bypassing the need for an OAuth dance. Clients
    SHOULD NOT rely on this in production.

12. Example wire transcripts

    The following examples are not normative ("\" indicates a line was
    wrapped).

12.1. WebFinger

    In application-first, an in-browser application might issue the
    following request, using XMLHttpRequest and CORS:


de Jong                                                        [Page 12]

        
Internet-Draft              remoteStorage                  December 2014



         GET /.well-known/webfinger?resource=acct:michiel@michielbdejon\
g.com HTTP/1.1
         Host: michielbdejong.com

    and the server's response might look like this:

         HTTP/1.1 200 OK
         Access-Control-Allow-Origin: *
         Access-Control-Allow-Methods: GET
         Access-Control-Allow-Headers: If-Match, If-None-Match
         Access-Control-Expose-Headers: ETag, Content-Length
         Content-Type: application/jrd+json

         {
           "links":[{
             "href": "https://michielbdejong.com:7678/inbox",
             "rel": "post-me-anything"
           }, {
             "href": "https://michielbdejong.com/me.jpg",
             "rel": "avatar"
           }, {
             "href": "https://3pp.io:4439/storage/michiel",
             "rel": "remotestorage",
             "properties": {
               "http://remotestorage.io/spec/version": "draft-dejong-re\
motestorage-04",
               "http://tools.ietf.org/html/rfc6749#section-4.2": "https\
://3pp.io:4439/oauth/michiel",
               "http://tools.ietf.org/html/rfc6750#section-2.3": false,
               "http://tools.ietf.org/html/rfc7233": false,
               "http://remotestorage.io/spec/web-authoring": false
             }
           }]
         }

12.2. OAuth dialog form

    Once the in-browser application has discovered the server's OAuth
    end-point, it will typically redirect the user to this URL, in
    order to obtain a bearer token. Say the application is hosted on
    https://drinks-unhosted.5apps.com/ and wants read-write access to
    the account's "myfavoritedrinks" scope:


de Jong                                                        [Page 13]

        
Internet-Draft              remoteStorage                  December 2014



        GET /oauth/michiel?redirect_uri=https%3A%2F%2Fdrinks-unhosted.5\
apps.com%2F&scope=myfavoritedrinks%3Arw&client_id=https%3A%2F%2Fdrinks-\
unhosted.5apps.com&response_type=token HTTP/1.1
        Host: 3pp.io

    The server's response might look like this (truncated for brevity):

        HTTP/1.1 200 OK

        <!DOCTYPE html>
        <html lang="en">
          <head>
            <title>Allow access?</title>
        ...

12.3. OAuth dialog form submission

    When the user submits the form, the request would look something
    like this:

        POST /oauth HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://3pp.io:4439
        Content-Type: application/x-www-form-urlencoded
        Referer: https://3pp.io:4439/oauth/michiel?redirect_uri=https%3\
A%2F%2Fdrinks-unhosted.5apps.com%2F&scope=myfavoritedrinks%3Arw&client_\
id=https%3A%2F%2Fdrinks-unhosted.5apps.com&response_type=token

        client_id=https%3A%2F%2Fdrinks-unhosted.5apps.com&redirect_uri=\
https%3A%2F%2Fdrinks-unhosted.5apps.com%2F&response_type=token&scope=my\
favoritedrinks%3Arw&state=&username=michiel&password=something&allow=Al\
low

    To which the server could respond with a 302 redirect, back to the
    origin of the requesting application:

        HTTP/1.1 302 Found
        Location:https://drinks-unhosted.5apps.com/#access_token=j2YnGt\
XjzzzHNjkd1CJxoQubA1o%3D&token_type=bearer&state=

12.4. OPTIONS preflight



de Jong                                                        [Page 14]

        
Internet-Draft              remoteStorage                  December 2014


    When an in-browser application makes a cross-origin request which
    may affect the server-state, the browser will make a preflight
    request first, with the OPTIONS verb, for instance:

        OPTIONS /storage/michiel/myfavoritedrinks/ HTTP/1.1
        Host: 3pp.io:4439
        Access-Control-Request-Method: GET
        Origin: https://drinks-unhosted.5apps.com
        Access-Control-Request-Headers: Authorization
        Referer: https://drinks-unhosted.5apps.com/

    To which the server can for instance respond:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Access-Control-Allow-Methods: GET, PUT, DELETE
        Access-Control-Allow-Headers: Authorization, Content-Length, Co\
ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match

12.5. Initial PUT

    An initial PUT may contain an 'If-None-Match: *' header, like this:

        PUT /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Content-Length: 91
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-None-Match: *

        {"name":"test","@context":"http://remotestorage.io/spec/modules\
/myfavoritedrinks/drink"}

    And the server may respond with either a 201 Created or a 200 OK
    status:

        HTTP/1.1 201 Created
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694045000"

12.6. Subsequent PUT


de Jong                                                        [Page 15]

        
Internet-Draft              remoteStorage                  December 2014



    A subsequent PUT may contain an 'If-Match' header referring to the
    ETag previously returned, like this:

        PUT /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Content-Length: 91
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-Match: "1382694045000"

        {"name":"test", "updated":true, "@context":"http://remotestorag\
e.io/spec/modules/myfavoritedrinks/drink"}

    And the server may respond with a 412 Conflict or a 200 OK status:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

12.7. GET

    A GET request would also include the bearer token, and optionally
    an If-None-Match header:

        GET /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Referer: https://drinks-unhosted.5apps.com/?
        If-None-Match: "1382694045000", "1382694048000"

    And the server may respond with a 304 Not Modified status:

        HTTP/1.1 304 Not Modified
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

    Or a 200 OK status, plus a response body:

        HTTP/1.1 200 OK


de Jong                                                        [Page 16]

        
Internet-Draft              remoteStorage                  December 2014


        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Content-Type: application/json; charset=UTF-8
        Content-Length: 106
        ETag: "1382694048000"
        Expires: 0

        {"name":"test", "updated":true, "@context":"http://remotestora\
ge.io/spec/modules/myfavoritedrinks/drink"}

    If the GET URL would have been "/storage/michiel/myfavoritedrinks/",
    a 200 OK response would have a folder description as the response
    body:

        HTTP/1.1 200 OK
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        Content-Type: application/ld+json
        Content-Length: 171
        ETag: "1382694048000"
        Expires: 0

        {"@context":"http://remotestorage.io/spec/folder-version","ite\
ms":{"test":{"ETag":"1382694048000","Content-Type":"application/json; \
charset=UTF-8","Content-Length":106}}}

    If the GET URL would have been a non-existing document like
    "/storage/michiel/myfavoritedrinks/x", the response would have a 404
    Not Found status, and no ETag header:

        HTTP/1.1 404 Not Found
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com

12.8. DELETE

    A DELETE request may look like this:

        DELETE /storage/michiel/myfavoritedrinks/test HTTP/1.1
        Host: 3pp.io:4439
        Origin: https://drinks-unhosted.5apps.com
        Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
        Content-Type: application/json; charset=UTF-8
        Referer: https://drinks-unhosted.5apps.com/?
        If-Match: "1382694045000"



de Jong                                                        [Page 17]

        
Internet-Draft              remoteStorage                  December 2014


    And the server may respond with a 412 Conflict or a 200 OK status:

        HTTP/1.1 412 Conflict
        Access-Control-Allow-Origin: https://drinks-unhosted.5apps.com
        ETag: "1382694048000"

13. Distributed versioning

    This section is non-normative, and is intended to explain some of
    the design choices concerning ETags and folder listings. At the
    same time it will hopefully help readers who intend to develop an
    application that uses remoteStorage as its per-user data storage.
    When multiple clients have read/write access to the same document,
    versioning conflicts may occur. For instance, client A may make
    a PUT request that changes the document from version 1 to version
    2, after which client B may make a PUT request attempting to change
    the same document from version 1 to version 3.

    In this case, client B can add an 'If-Match: "1"' header, which
    would trigger a 412 Conflict response code, since the current
    version ("2") does not match the version required as a condition by
    the header If-Match header ("1").

    Client B is now aware of the conflict, and may consult the user,
    saying the update to version 3 failed. The user may then choose,
    through the user interface of client B, whether version 2 or
    version 3 should be kept, or maybe the document should be reverted
    on the server to version 1, or a merged version 4 is needed. Client
    B may then make a request that puts the document to the version the
    user wishes; this time setting an 'If-Match: "2"' header instead.

    Both client A and client B would periodically poll the root
    folder of each scope they have access to, to see if the version
    of the root folder changed. If it did, then one of the versions
    listed in there will necessarily have changed, and the client can
    make a GET request to that child folder or document, to obtain
    its latest version.

    Because an update in a document will result in a version change of
    its containing folder, and that change will propagate all the way
    to the root folder, it is not necessary to poll each document for
    changes individually.



de Jong                                                        [Page 18]

        
Internet-Draft              remoteStorage                  December 2014


    As an example, the root folder may contain 10 directories,
    each of which contain 10 directories, which each contain 10
    documents, so their paths would be for instance '/0/0/1', '/0/0/2',
    etcetera. Then one GET request to the root folder '/' will be
    enough to know if any of these 1000 documents has changed.

    Say document '/7/9/2' has changed; then the GET request to '/' will
    come back with a different ETag, and entry '7/' will have a
    different value in its JSON content. The client could then request
    '/7/', '/7/9/', and '/7/9/2' to narrow down the one document that
    caused the root folder's ETag to change.

    Note that the remoteStorage server does not get involved in the
    conflict resolution. It keeps the canonical current version at all
    times, and allows clients to make conditional GET and PUT requests,
    but it is up to whichever client discovers a given version
    conflict, to resolve it.

14. Security Considerations

    To prevent man-in-the-middle attacks, the use of https instead of
    http is important for both the interface itself and all end-points
    involved in WebFinger, OAuth, and (if present) the storage-first
    application launch dashboard.

    A malicious party could link to an application, but specifying a
    remoteStorage account address that it controls, thus tricking the
    user into using a trusted application to send sensitive data to the
    wrong remoteStorage server. To mitigate this, applications SHOULD
    clearly display to which remoteStorage server they are sending the
    user's data.

    Applications could request scopes that the user did not intend to
    give access to. The user SHOULD always be prompted to carefully
    review which scopes an application is requesting.

    An application may upload malicious html pages and then trick the
    user into visiting them, or upload malicious client-side scripts,
    that take advantage of being hosted on the user's domain name. The
    origin on which the remoteStorage server has its interface SHOULD
    therefore NOT be used for anything else, and the user SHOULD be
    warned not to visit any web pages on that origin. In particular, the
    OAuth dialog and launch dashboard or token revokation interface


de Jong                                                        [Page 19]

        
Internet-Draft              remoteStorage                  December 2014


    SHOULD be on a different origin than the remoteStorage interface.

    Where the use of bearer tokens is impractical, a user may choose to
    store documents on hard-to-guess URLs whose path after
    <storage_root> starts with '/public/', while sharing this URL only
    with the intended audience. That way, only parties who know the
    document's hard-to-guess URL, can access it. The server SHOULD
    therefore make an effort to detect and stop brute-force attacks that
    attempt to guess the location of such documents.

    The server SHOULD also detect and stop denial-of-service attacks
    that aim to overwhelm its interface with too much traffic.

15. IANA Considerations

    This document registers the 'remotestorage' link relation, as well
    as the following WebFinger properties:
      * "http://remotestorage.io/spec/version"
      * "http://tools.ietf.org/html/rfc6749#section-4.2"
      * "http://tools.ietf.org/html/rfc6750#section-2.3"
      * "http://tools.ietf.org/html/rfc7233"
      * "http://remotestorage.io/spec/web-authoring"

16. Acknowledgements

    The authors would like to thank everybody who contributed to the
    development of this protocol, including Kenny Bentley, Javier Diaz,
    Daniel Groeber, Bjarni Runar, Jan Wildeboer, Charles Schultz, Peter
    Svensson, Valer Mischenko, Michiel Leenaars, Jan-Christoph
    Borchardt, Garret Alfert, Sebastian Kippe, Max Wiehle, Melvin
    Carvalho, Martin Stadler, Geoffroy Couprie, Niklas Cathor, Marco
    Stahl, James Coglan, Ken Eucker, Daniel Brolund, elf Pavlik, Nick
    Jennings, Markus Sabadello, Steven te Brinke, Matthias Treydte,
    Rick van Rein, Mark Nottingham, Julian Reschke, and Markus
    Lanthaler, among many others.

17. References

17.1. Normative References

    [WORDS]
        Bradner, S., "Key words for use in RFCs to Indicate Requirement
        Levels", BCP 14, RFC 2119, March 1997.


de Jong                                                        [Page 20]

        
Internet-Draft              remoteStorage                  December 2014



    [IRI]
        Duerst, M., "Internationalized Resource Identifiers (IRIs)",
        RFC 3987, January 2005.

    [WEBFINGER]
        Jones, P., Salguerio, G., Jones, M, and Smarr, J.,
        "WebFinger", RFC7033, September 2013.

    [OAUTH]
        "Section 4.2: Implicit Grant", in: Hardt, D. (ed), "The OAuth
        2.0 Authorization Framework", RFC6749, October 2012.

17.2. Informative References

    [HTTPS]
        Rescorla, E., "HTTP Over TLS", RFC2818, May 2000.

    [HTTP]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Semantics and Content", RFC7231, June 2014.

    [COND]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Conditional Requests", RFC7232, June 2014.

    [RANGE]
        Fielding et al., "Hypertext Transfer Protocol (HTTP/1.1):
        Conditional Requests", RFC7233, June 2014.

    [SPDY]
        Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", http://
        www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1,
        September 2013.

    [JSON-LD]
        M. Sporny, G. Kellogg, M. Lanthaler, "JSON-LD 1.0", W3C
        Proposed Recommendation,
        http://www.w3.org/TR/2014/REC-json-ld-20140116/, January 2014.

    [CORS]
        van Kesteren, Anne (ed), "Cross-Origin Resource Sharing --
        W3C Candidate Recommendation 29 January 2013",


de Jong                                                        [Page 21]

        
Internet-Draft              remoteStorage                  December 2014


        http://www.w3.org/TR/cors/, January 2013.

    [MANIFEST]
        Mozilla Developer Network (ed), "App manifest -- Revision
        330541", https://developer.mozilla.org/en-
        US/Apps/Build/Manifest$revision/566677, April 2014.

    [DATASTORE]
        "WebAPI/DataStore", MozillaWiki, retrieved May 2014.
        https://wiki.mozilla.org/WebAPI/DataStore#Manifest

    [KERBEROS]
        C. Neuman et al., "The Kerberos Network Authentication Service
        (V5)", RFC4120, July 2005.

    [BEARER]
        M. Jones, D. Hardt, "The OAuth 2.0 Authorization Framework:
        Bearer Token Usage", RFC6750, October 2012.

    [AUTHORING]
        "Using remoteStorage for web authoring", reSite wiki, retrieved
        September 2014. https://github.com/michielbdejong/resite/wiki
        /Using-remoteStorage-for-web-authoring

18. Authors' addresses

    Michiel B. de Jong
    IndieHosters

    Email: michiel@michielbdejong.com


    F. Kooman
    (independent)

    Email: fkooman@tuxed.net









de Jong                                                        [Page 22]

        Html markup produced by rfcmarkup 1.111, available from https://tools.ietf.org/tools/rfcmarkup/ ================================================ FILE: test/test-pages/invalid-attributes/expected-metadata.json ================================================ { "title": "Lorem Ipsum", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/invalid-attributes/expected.html ================================================

        Lorem ipsum dolor sit amet, consectetur adipiscing elit.

        ================================================ FILE: test/test-pages/invalid-attributes/source.html ================================================ Lorem Ipsum
        Lorem ipsum dolor sit amet, consectetur adipiscing elit.
        ================================================ FILE: test/test-pages/js-link-replacement/expected-metadata.json ================================================ { "title": "Replace javascript: links", "byline": null, "dir": null, "excerpt": "abc", "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/js-link-replacement/expected.html ================================================

        abc

        def

        ghi
        ================================================ FILE: test/test-pages/js-link-replacement/source.html ================================================ Replace javascript: links

        abc

        def

        ghi         ================================================ FILE: test/test-pages/keep-images/expected-metadata.json ================================================ { "title": "Inside the Deep Web Drug Lab", "byline": "Joseph Cox", "dir": null, "excerpt": "Welcome to DoctorX’s Barcelona lab, where the drugs you bought online are tested for safety and purity. No questions ask…", "siteName": "Medium", "publishedTime": "2015-03-27T13:07:55.096Z", "readerable": true } ================================================ FILE: test/test-pages/keep-images/expected.html ================================================

        Welcome to DoctorX’s Barcelona lab, where the drugs you bought online are tested for safety and purity. No questions asked.

        Standing at a table in a chemistry lab in Barcelona, Cristina Gil Lladanosa tears open a silver, smell-proof protective envelope. She slides out a transparent bag full of crystals. Around her, machines whir and hum, and other researchers mill around in long, white coats.

        She is holding the lab’s latest delivery of a drug bought from the “deep web,” the clandestine corner of the internet that isn’t reachable by normal search engines, and is home to some sites that require special software to access. Labeled as MDMA (the street term is ecstasy), this sample has been shipped from Canada. Lladanosa and her colleague Iván Fornís Espinosa have also received drugs, anonymously, from people in China, Australia, Europe and the United States.

        “Here we have speed, MDMA, cocaine, pills,” Lladanosa says, pointing to vials full of red, green, blue and clear solutions sitting in labeled boxes.

        Cristina Gil Lladanosa, at the Barcelona testing lab | photo by Joan Bardeletti

        Since 2011, with the launch of Silk Road, anybody has been able to safely buy illegal drugs from the deep web and have them delivered to their door. Though the FBI shut down that black market in October 2013, other outlets have emerged to fill its role. For the last 10 months the lab at which Lladanosa and Espinosa work has offered a paid testing service of those drugs. By sending in samples for analysis, users can know exactly what it is they are buying, and make a more informed decision about whether to ingest the substance. The group, called Energy Control, which has being running “harm reduction” programs since 1999, is the first to run a testing service explicitly geared towards verifying those purchases from the deep web.

        Before joining Energy Control, Lladanosa briefly worked at a pharmacy, whereas Espinosa spent 14 years doing drug analysis. Working at Energy Control is “more gratifying,” and “rewarding” than her previous jobs, Lladanosa told me. They also receive help from a group of volunteers, made up of a mixture of “squatters,” as Espinosa put it, and medical students, who prepare the samples for testing.

        After weighing out the crystals, aggressively mixing it with methanol until dissolved, and delicately pouring the liquid into a tiny brown bottle, Lladanosa, a petite woman who is nearly engulfed by her lab coat, is now ready to test the sample. She loads a series of three trays on top of a large white appliance sitting on a table, called a gas chromatograph (GC). A jungle of thick pipes hang from the lab’s ceiling behind it.

        Photo by Joan Bardeletti

        “Chromatography separates all the substances,” Lladanosa says as she loads the machine with an array of drugs sent from the deep web and local Spanish users. It can tell whether a sample is pure or contaminated, and if the latter, with what.

        Rushes of hot air blow across the desk as the gas chromatograph blasts the sample at 280 degrees Celsius. Thirty minutes later the machine’s robotic arm automatically moves over to grip another bottle. The machine will continue cranking through the 150 samples in the trays for most of the work week.

        Photo by Joan Bardeletti

        To get the drugs to Barcelona, a user mails at least 10 milligrams of a substance to the offices of the Asociación Bienestar y Desarrollo, the non-government organization that oversees Energy Control. The sample then gets delivered to the testing service’s laboratory, at the Barcelona Biomedical Research Park, a futuristic, seven story building sitting metres away from the beach. Energy Control borrows its lab space from a biomedical research group for free.

        The tests cost 50 Euro per sample. Users pay, not surprisingly, with Bitcoin. In the post announcing Energy Control’s service on the deep web, the group promised that “All profits of this service are set aside of maintenance of this project.”

        About a week after testing, those results are sent in a PDF to an email address provided by the anonymous client.

        “The process is quite boring, because you are in a routine,” Lladanosa says. But one part of the process is consistently surprising: that moment when the results pop up on the screen. “Every time it’s something different.” For instance, one cocaine sample she had tested also contained phenacetin, a painkiller added to increase the product’s weight; lidocaine, an anesthetic that numbs the gums, giving the impression that the user is taking higher quality cocaine; and common caffeine.

        The deep web drug lab is the brainchild of Fernando Caudevilla, a Spanish physician who is better known as “DoctorX” on the deep web, a nickname given to him by his Energy Control co-workers because of his earlier writing about the history, risks and recreational culture of MDMA. In the physical world, Caudevilla has worked for over a decade with Energy Control on various harm reduction focused projects, most of which have involved giving Spanish illegal drug users medical guidance, and often writing leaflets about the harms of certain substances.

        Fernando Caudevilla, AKA DoctorX. Photo: Joseph Cox

        Caudevilla first ventured into Silk Road forums in April 2013. “I would like to contribute to this forum offering professional advice in topics related to drug use and health,” he wrote in an introductory post, using his DoctorX alias. Caudevilla offered to provide answers to questions that a typical doctor is not prepared, or willing, to respond to, at least not without a lecture or a judgment. “This advice cannot replace a complete face-to-face medical evaluation,” he wrote, “but I know how difficult it can be to talk frankly about these things.”

        The requests flooded in. A diabetic asked what effect MDMA has on blood sugar; another what the risks of frequent psychedelic use were for a young person. Someone wanted to know whether amphetamine use should be avoided during lactation. In all, Fernando’s thread received over 50,000 visits and 300 questions before the FBI shut down Silk Road.

        “He’s amazing. A gift to this community,” one user wrote on the Silk Road 2.0 forum, a site that sprang up after the original. “His knowledge is invaluable, and never comes with any judgment.” Up until recently, Caudevilla answered questions on the marketplace “Evolution.” Last week, however, the administrators of that site pulled a scam, shutting the market down and escaping with an estimated $12 million worth of Bitcoin.

        Caudevilla’s transition from dispensing advice to starting up a no-questions-asked drug testing service came as a consequence of his experience on the deep web. He’d wondered whether he could help bring more harm reduction services to a marketplace without controls. The Energy Control project, as part of its mandate of educating drug users and preventing harm, had already been carrying out drug testing for local Spanish users since 2001, at music festivals, night clubs, or through a drop-in service at a lab in Madrid.

        “I thought, we are doing this in Spain, why don’t we do an international drug testing service?” Caudevilla told me when I visited the other Energy Control lab, in Madrid. Caudevilla, a stocky character with ear piercings and short, shaved hair, has eyes that light up whenever he discusses the world of the deep web. Later, via email, he elaborated that it was not a hard sell. “It was not too hard to convince them,” he wrote me. Clearly, Energy Control believed that the reputation he had earned as an unbiased medical professional on the deep web might carry over to the drug analysis service, where one needs to establish “credibility, trustworthiness, [and] transparency,” Caudevilla said. “We could not make mistakes,” he added.

        Photo: Joseph Cox

        While the Energy Control lab in Madrid lab only tests Spanish drugs from various sources, it is the Barcelona location which vets the substances bought in the shadowy recesses of of the deep web. Caudevilla no longer runs it, having handed it over to his colleague Ana Muñoz. She maintains a presence on the deep web forums, answers questions from potential users, and sends back reports when they are ready.

        The testing program exists in a legal grey area. The people who own the Barcelona lab are accredited to experiment with and handle drugs, but Energy Control doesn’t have this permission itself, at least not in writing.

        “We have a verbal agreement with the police and other authorities. They already know what we are doing,” Lladanosa tells me. It is a pact of mutual benefit. Energy Control provides the police with information on batches of drugs in Spain, whether they’re from the deep web or not, Espinosa says. They also contribute to the European Monitoring Centre for Drugs and Drug Addiction’s early warning system, a collaboration that attempts to spread information about dangerous drugs as quickly as possible.

        By the time of my visit in February, Energy Control had received over 150 samples from the deep web and have been receiving more at a rate of between 4 and 8 a week. Traditional drugs, such as cocaine and MDMA, make up about 70 percent of the samples tested, but the Barcelona lab has also received samples of the prescription pill codeine, research chemicals and synthetic cannabinoids, and even pills of Viagra.

        Photo by Joan Bardeletti

        So it’s fair to make a tentative judgement on what people are paying for on the deep web. The verdict thus far? Overall, drugs on the deep web appear to be of much higher quality than those found on the street.

        “In general, the cocaine is amazing,” says Caudevilla, saying that the samples they’ve seen have purities climbing towards 80 or 90 percent, and some even higher. To get an idea of how unusual this is, take a look at the United Nations Office on Drugs and Crime World Drug Report 2014, which reports that the average quality of street cocaine in Spain is just over 40 percent, while in the United Kingdom it is closer to 30 percent.“We have found 100 percent [pure] cocaine,” he adds. “That’s really, really strange. That means that, technically, this cocaine has been purified, with clandestine methods.”

        Naturally, identifying vendors who sell this top-of-the-range stuff is one of the reasons that people have sent samples to Energy Control. Caudevilla was keen to stress that, officially, Energy Control’s service “is not intended to be a control of drug quality,” meaning a vetting process for identifying the best sellers, but that is exactly how some people have been using it.

        As one buyer on the Evolution market, elmo666, wrote to me over the site’s messaging system, “My initial motivations were selfish. My primary motivation was to ensure that I was receiving and continue to receive a high quality product, essentially to keep the vendor honest as far as my interactions with them went.”

        Vendors on deep web markets advertise their product just like any other outlet does, using flash sales, gimmicky giveaways and promises of drugs that are superior to those of their competitors. The claims, however, can turn out to be empty: despite the test results that show that deep web cocaine vendors typically sell product that is of a better quality than that found on the street, in plenty of cases, the drugs are nowhere near as pure as advertised.

        “You won’t be getting anything CLOSE to what you paid for,” one user complained about the cocaine from ‘Mirkov’, a vendor on Evolution. “He sells 65% not 95%.”

        Photo by Joan Bardeletti

        Despite the prevalence of people using the service to gauge the quality of what goes up their nose, many users send samples to Energy Control in the spirit of its original mission: keeping themselves alive and healthy. The worst case scenario from drugs purchased on the deep web is, well the worst case. That was the outcome when Patrick McMullen, a 17-year-old Scottish student, ingested half a gram of MDMA and three tabs of LSD, reportedly purchased from the Silk Road. While talking to his friends on Skype, his words became slurred and he passed out. Paramedics could not revive him. The coroner for that case, Sherrif Payne, who deemed the cause of death ecstasy toxicity, told The Independent “You never know the purity of what you are taking and you can easily come unstuck.”

        ScreamMyName, a deep web user who has been active since the original Silk Road, wants to alert users to the dangerous chemicals that are often mixed with drugs, and is using Energy Control as a means to do so.

        “We’re at a time where some vendors are outright sending people poison. Some do it unknowingly,” ScreamMyName told me in an encrypted message. “Cocaine production in South America is often tainted with either levamisole or phenacetine. Both poison to humans and both with severe side effects.”

        In the case of Levamisole, those prescribing it are often not doctors but veterinarians, as Levamisole is commonly used on animals, primarily for the treatment of worms. If ingested by humans it can lead to cases of extreme eruptions of the skin, as documented in a study from researchers at the University of California, San Francisco. But Lladanosa has found Levamisole in cocaine samples; dealers use it to increase the product weight, allowing them to stretch their batch further for greater profit — and also, she says, because Levamisole has a strong stimulant effect.

        “It got me sick as fuck,” Dr. Feel, an Evolution user, wrote on the site’s forums after consuming cocaine that had been cut with 23 percent Levamisole, and later tested by Energy Control. “I was laid up in bed for several days because of that shit. The first night I did it, I thought I was going to die. I nearly drove myself to the ER.”

        “More people die because of tainted drugs than the drugs themselves,” Dr. Feel added. “It’s the cuts and adulterants that are making people sick and killing them.”

        Photo by Joan Bardeletti

        The particular case of cocaine cut with Levamisole is one of the reasons that ScreamMyName has been pushing for more drug testing on the deep web markets. “I recognize that drug use isn’t exactly healthy, but why exacerbate the problem?” he told me when I contacted him after his post. “[Energy Control] provides a way for users to test the drugs they’ll use and for these very users to know what it is they’re putting in their bodies. Such services are in very short supply.”

        After sending a number of Energy Control tests himself, ScreamMyName started a de facto crowd-sourcing campaign to get more drugs sent to the lab, and then shared the results, after throwing in some cash to get the ball rolling. He set up a Bitcoin wallet, with the hope that users might chip in to fund further tests. At the time of writing, the wallet has received a total of 1.81 bitcoins; around $430 at today’s exchange rates.

        In posts to the Evolution community, ScreamMyName pitched this project as something that will benefit users and keep drug dealer honest. “When the funds build up to a point where we can purchase an [Energy Control] test fee, we’ll do a US thread poll for a few days and try to cohesively decide on what vendor to test,” he continued.

        Photo by Joan Bardeletti

        Other members of the community have been helping out, too. PlutoPete, a vendor from the original Silk Road who sold cannabis seeds and other legal items, has provided ScreamMyName with packaging to safely send the samples to Barcelona. “A box of baggies, and a load of different moisture barrier bags,” PlutoPete told me over the phone. “That’s what all the vendors use.”

        It’s a modest program so far. ScreamMyName told me that so far he had gotten enough public funding to purchase five different Energy Control tests, in addition to the ten or so he’s sent himself so far. “The program created is still in its infancy and it is growing and changing as we go along but I have a lot of faith in what we’re doing,” he says.

        But the spirit is contagious: elmo666, the other deep web user testing cocaine, originally kept the results of the drug tests to himself, but he, too, saw a benefit to distributing the data. “It is clear that it is a useful service to other users, keeping vendors honest and drugs (and their users) safe,” he told me. He started to report his findings to others on the forums, and then created a thread with summaries of the test results, as well as comments from the vendors if they provided it. Other users were soon basing their decisions on what to buy on elmo666‘s tests.

        “I’m defo trying the cola based on the incredibly helpful elmo and his energy control results and recommendations,” wrote user jayk1984. On top of this, elmo666 plans to launch an independent site on the deep web that will collate all of these results, which should act as a resource for users of all the marketplaces.

        As word of elmo666's efforts spread, he began getting requests from drug dealers who wanted him to use their wares for testing. Clearly, they figured that a positive result from Energy Control would be a fantastic marketing tool to draw more customers. They even offered elmo666 free samples. (He passed.)

        Meanwhile, some in the purchasing community are arguing that those running markets on the deep web should be providing quality control themselves. PlutoPete told me over the phone that he had been in discussions about this with Dread Pirate Roberts, the pseudonymous owner of the original Silk Road site. “We [had been] talking about that on a more organized basis on Silk Road 1, doing lots of anonymous buys to police each category. But of course they took the thing [Silk Road] down before we got it properly off the ground,” he lamented.

        But perhaps it is best that the users, those who are actually consuming the drugs, remain in charge of shaming dealers and warning each other. “It’s our responsibility to police the market based on reviews and feedback,” elmo666 wrote in an Evolution forum post. It seems that in the lawless space of the deep web, where everything from child porn to weapons are sold openly, users have cooperated in an organic display of self-regulation to stamp out those particular batches of drugs that are more likely to harm users.

        “That’s always been the case with the deep web,” PlutoPete told me. Indeed, ever since Silk Road, a stable of the drug markets has been the review system, where buyers can leave a rating and feedback for vendors, letting others know about the reliability of the seller. But DoctorX’s lab, rigorously testing the products with scientific instruments, takes it a step further.

        Photo by Joan Bardeletti

        “In the white market, they have quality control. In the dark market, it should be the same,” Cristina Gil Lladanosa says to me before I leave the Barcelona lab.

        A week after I visit the lab, the results of the MDMA arrive in my inbox: it is 85 percent pure, with no indications of other active ingredients. Whoever ordered that sample from the digital shelves of the deep web, and had it shipped to their doorstep in Canada, got hold of some seriously good, and relatively safe drugs. And now they know it.

        Top photo by Joan Bardeletti

        Follow Backchannel: Twitter |Facebook

        ================================================ FILE: test/test-pages/keep-images/source.html ================================================ Inside the Deep Web Drug Lab — Backchannel — Medium
        Ready to publish?
        Change the story’s title, subtitle, and visibility as needed
        Joseph Cox
        14 min read

        Inside the Deep Web Drug Lab


        Welcome to DoctorX’s Barcelona lab, where the drugs you bought online are tested for safety and purity. No questions asked.


        Standing at a table in a chemistry lab in Barcelona, Cristina Gil Lladanosa tears open a silver, smell-proof protective envelope. She slides out a transparent bag full of crystals. Around her, machines whir and hum, and other researchers mill around in long, white coats.

        She is holding the lab’s latest delivery of a drug bought from the “deep web,” the clandestine corner of the internet that isn’t reachable by normal search engines, and is home to some sites that require special software to access. Labeled as MDMA (the street term is ecstasy), this sample has been shipped from Canada. Lladanosa and her colleague Iván Fornís Espinosa have also received drugs, anonymously, from people in China, Australia, Europe and the United States.

        “Here we have speed, MDMA, cocaine, pills,” Lladanosa says, pointing to vials full of red, green, blue and clear solutions sitting in labeled boxes.

        Cristina Gil Lladanosa, at the Barcelona testing lab | photo by Joan Bardeletti

        Since 2011, with the launch of Silk Road, anybody has been able to safely buy illegal drugs from the deep web and have them delivered to their door. Though the FBI shut down that black market in October 2013, other outlets have emerged to fill its role. For the last 10 months the lab at which Lladanosa and Espinosa work has offered a paid testing service of those drugs. By sending in samples for analysis, users can know exactly what it is they are buying, and make a more informed decision about whether to ingest the substance. The group, called Energy Control, which has being running “harm reduction” programs since 1999, is the first to run a testing service explicitly geared towards verifying those purchases from the deep web.

        Before joining Energy Control, Lladanosa briefly worked at a pharmacy, whereas Espinosa spent 14 years doing drug analysis. Working at Energy Control is “more gratifying,” and “rewarding” than her previous jobs, Lladanosa told me. They also receive help from a group of volunteers, made up of a mixture of “squatters,” as Espinosa put it, and medical students, who prepare the samples for testing.

        After weighing out the crystals, aggressively mixing it with methanol until dissolved, and delicately pouring the liquid into a tiny brown bottle, Lladanosa, a petite woman who is nearly engulfed by her lab coat, is now ready to test the sample. She loads a series of three trays on top of a large white appliance sitting on a table, called a gas chromatograph (GC). A jungle of thick pipes hang from the lab’s ceiling behind it.

        Photo by Joan Bardeletti

        “Chromatography separates all the substances,” Lladanosa says as she loads the machine with an array of drugs sent from the deep web and local Spanish users. It can tell whether a sample is pure or contaminated, and if the latter, with what.

        Rushes of hot air blow across the desk as the gas chromatograph blasts the sample at 280 degrees Celsius. Thirty minutes later the machine’s robotic arm automatically moves over to grip another bottle. The machine will continue cranking through the 150 samples in the trays for most of the work week.

        Photo by Joan Bardeletti

        To get the drugs to Barcelona, a user mails at least 10 milligrams of a substance to the offices of the Asociación Bienestar y Desarrollo, the non-government organization that oversees Energy Control. The sample then gets delivered to the testing service’s laboratory, at the Barcelona Biomedical Research Park, a futuristic, seven story building sitting metres away from the beach. Energy Control borrows its lab space from a biomedical research group for free.

        The tests cost 50 Euro per sample. Users pay, not surprisingly, with Bitcoin. In the post announcing Energy Control’s service on the deep web, the group promised that “All profits of this service are set aside of maintenance of this project.”

        About a week after testing, those results are sent in a PDF to an email address provided by the anonymous client.

        “The process is quite boring, because you are in a routine,” Lladanosa says. But one part of the process is consistently surprising: that moment when the results pop up on the screen. “Every time it’s something different.” For instance, one cocaine sample she had tested also contained phenacetin, a painkiller added to increase the product’s weight; lidocaine, an anesthetic that numbs the gums, giving the impression that the user is taking higher quality cocaine; and common caffeine.

        The deep web drug lab is the brainchild of Fernando Caudevilla, a Spanish physician who is better known as “DoctorX” on the deep web, a nickname given to him by his Energy Control co-workers because of his earlier writing about the history, risks and recreational culture of MDMA. In the physical world, Caudevilla has worked for over a decade with Energy Control on various harm reduction focused projects, most of which have involved giving Spanish illegal drug users medical guidance, and often writing leaflets about the harms of certain substances.

        Fernando Caudevilla, AKA DoctorX. Photo: Joseph Cox

        Caudevilla first ventured into Silk Road forums in April 2013. “I would like to contribute to this forum offering professional advice in topics related to drug use and health,” he wrote in an introductory post, using his DoctorX alias. Caudevilla offered to provide answers to questions that a typical doctor is not prepared, or willing, to respond to, at least not without a lecture or a judgment. “This advice cannot replace a complete face-to-face medical evaluation,” he wrote, “but I know how difficult it can be to talk frankly about these things.”

        The requests flooded in. A diabetic asked what effect MDMA has on blood sugar; another what the risks of frequent psychedelic use were for a young person. Someone wanted to know whether amphetamine use should be avoided during lactation. In all, Fernando’s thread received over 50,000 visits and 300 questions before the FBI shut down Silk Road.

        “He’s amazing. A gift to this community,” one user wrote on the Silk Road 2.0 forum, a site that sprang up after the original. “His knowledge is invaluable, and never comes with any judgment.” Up until recently, Caudevilla answered questions on the marketplace “Evolution.” Last week, however, the administrators of that site pulled a scam, shutting the market down and escaping with an estimated $12 million worth of Bitcoin.

        Caudevilla’s transition from dispensing advice to starting up a no-questions-asked drug testing service came as a consequence of his experience on the deep web. He’d wondered whether he could help bring more harm reduction services to a marketplace without controls. The Energy Control project, as part of its mandate of educating drug users and preventing harm, had already been carrying out drug testing for local Spanish users since 2001, at music festivals, night clubs, or through a drop-in service at a lab in Madrid.

        “I thought, we are doing this in Spain, why don’t we do an international drug testing service?” Caudevilla told me when I visited the other Energy Control lab, in Madrid. Caudevilla, a stocky character with ear piercings and short, shaved hair, has eyes that light up whenever he discusses the world of the deep web. Later, via email, he elaborated that it was not a hard sell. “It was not too hard to convince them,” he wrote me. Clearly, Energy Control believed that the reputation he had earned as an unbiased medical professional on the deep web might carry over to the drug analysis service, where one needs to establish “credibility, trustworthiness, [and] transparency,” Caudevilla said. “We could not make mistakes,” he added.

        Photo: Joseph Cox

        While the Energy Control lab in Madrid lab only tests Spanish drugs from various sources, it is the Barcelona location which vets the substances bought in the shadowy recesses of of the deep web. Caudevilla no longer runs it, having handed it over to his colleague Ana Muñoz. She maintains a presence on the deep web forums, answers questions from potential users, and sends back reports when they are ready.

        The testing program exists in a legal grey area. The people who own the Barcelona lab are accredited to experiment with and handle drugs, but Energy Control doesn’t have this permission itself, at least not in writing.

        “We have a verbal agreement with the police and other authorities. They already know what we are doing,” Lladanosa tells me. It is a pact of mutual benefit. Energy Control provides the police with information on batches of drugs in Spain, whether they’re from the deep web or not, Espinosa says. They also contribute to the European Monitoring Centre for Drugs and Drug Addiction’s early warning system, a collaboration that attempts to spread information about dangerous drugs as quickly as possible.

        By the time of my visit in February, Energy Control had received over 150 samples from the deep web and have been receiving more at a rate of between 4 and 8 a week. Traditional drugs, such as cocaine and MDMA, make up about 70 percent of the samples tested, but the Barcelona lab has also received samples of the prescription pill codeine, research chemicals and synthetic cannabinoids, and even pills of Viagra.

        Photo by Joan Bardeletti

        So it’s fair to make a tentative judgement on what people are paying for on the deep web. The verdict thus far? Overall, drugs on the deep web appear to be of much higher quality than those found on the street.

        “In general, the cocaine is amazing,” says Caudevilla, saying that the samples they’ve seen have purities climbing towards 80 or 90 percent, and some even higher. To get an idea of how unusual this is, take a look at the United Nations Office on Drugs and Crime World Drug Report 2014, which reports that the average quality of street cocaine in Spain is just over 40 percent, while in the United Kingdom it is closer to 30 percent.“We have found 100 percent [pure] cocaine,” he adds. “That’s really, really strange. That means that, technically, this cocaine has been purified, with clandestine methods.”

        Naturally, identifying vendors who sell this top-of-the-range stuff is one of the reasons that people have sent samples to Energy Control. Caudevilla was keen to stress that, officially, Energy Control’s service “is not intended to be a control of drug quality,” meaning a vetting process for identifying the best sellers, but that is exactly how some people have been using it.

        As one buyer on the Evolution market, elmo666, wrote to me over the site’s messaging system, “My initial motivations were selfish. My primary motivation was to ensure that I was receiving and continue to receive a high quality product, essentially to keep the vendor honest as far as my interactions with them went.”

        Vendors on deep web markets advertise their product just like any other outlet does, using flash sales, gimmicky giveaways and promises of drugs that are superior to those of their competitors. The claims, however, can turn out to be empty: despite the test results that show that deep web cocaine vendors typically sell product that is of a better quality than that found on the street, in plenty of cases, the drugs are nowhere near as pure as advertised.

        “You won’t be getting anything CLOSE to what you paid for,” one user complained about the cocaine from ‘Mirkov’, a vendor on Evolution. “He sells 65% not 95%.”

        Photo by Joan Bardeletti

        Despite the prevalence of people using the service to gauge the quality of what goes up their nose, many users send samples to Energy Control in the spirit of its original mission: keeping themselves alive and healthy. The worst case scenario from drugs purchased on the deep web is, well the worst case. That was the outcome when Patrick McMullen, a 17-year-old Scottish student, ingested half a gram of MDMA and three tabs of LSD, reportedly purchased from the Silk Road. While talking to his friends on Skype, his words became slurred and he passed out. Paramedics could not revive him. The coroner for that case, Sherrif Payne, who deemed the cause of death ecstasy toxicity, told The Independent “You never know the purity of what you are taking and you can easily come unstuck.”

        ScreamMyName, a deep web user who has been active since the original Silk Road, wants to alert users to the dangerous chemicals that are often mixed with drugs, and is using Energy Control as a means to do so.

        “We’re at a time where some vendors are outright sending people poison. Some do it unknowingly,” ScreamMyName told me in an encrypted message. “Cocaine production in South America is often tainted with either levamisole or phenacetine. Both poison to humans and both with severe side effects.”

        In the case of Levamisole, those prescribing it are often not doctors but veterinarians, as Levamisole is commonly used on animals, primarily for the treatment of worms. If ingested by humans it can lead to cases of extreme eruptions of the skin, as documented in a study from researchers at the University of California, San Francisco. But Lladanosa has found Levamisole in cocaine samples; dealers use it to increase the product weight, allowing them to stretch their batch further for greater profit — and also, she says, because Levamisole has a strong stimulant effect.

        “It got me sick as fuck,” Dr. Feel, an Evolution user, wrote on the site’s forums after consuming cocaine that had been cut with 23 percent Levamisole, and later tested by Energy Control. “I was laid up in bed for several days because of that shit. The first night I did it, I thought I was going to die. I nearly drove myself to the ER.”

        “More people die because of tainted drugs than the drugs themselves,” Dr. Feel added. “It’s the cuts and adulterants that are making people sick and killing them.”

        Photo by Joan Bardeletti

        The particular case of cocaine cut with Levamisole is one of the reasons that ScreamMyName has been pushing for more drug testing on the deep web markets. “I recognize that drug use isn’t exactly healthy, but why exacerbate the problem?” he told me when I contacted him after his post. “[Energy Control] provides a way for users to test the drugs they’ll use and for these very users to know what it is they’re putting in their bodies. Such services are in very short supply.”

        After sending a number of Energy Control tests himself, ScreamMyName started a de facto crowd-sourcing campaign to get more drugs sent to the lab, and then shared the results, after throwing in some cash to get the ball rolling. He set up a Bitcoin wallet, with the hope that users might chip in to fund further tests. At the time of writing, the wallet has received a total of 1.81 bitcoins; around $430 at today’s exchange rates.

        In posts to the Evolution community, ScreamMyName pitched this project as something that will benefit users and keep drug dealer honest. “When the funds build up to a point where we can purchase an [Energy Control] test fee, we’ll do a US thread poll for a few days and try to cohesively decide on what vendor to test,” he continued.

        Photo by Joan Bardeletti

        Other members of the community have been helping out, too. PlutoPete, a vendor from the original Silk Road who sold cannabis seeds and other legal items, has provided ScreamMyName with packaging to safely send the samples to Barcelona. “A box of baggies, and a load of different moisture barrier bags,” PlutoPete told me over the phone. “That’s what all the vendors use.”

        It’s a modest program so far. ScreamMyName told me that so far he had gotten enough public funding to purchase five different Energy Control tests, in addition to the ten or so he’s sent himself so far. “The program created is still in its infancy and it is growing and changing as we go along but I have a lot of faith in what we’re doing,” he says.

        But the spirit is contagious: elmo666, the other deep web user testing cocaine, originally kept the results of the drug tests to himself, but he, too, saw a benefit to distributing the data. “It is clear that it is a useful service to other users, keeping vendors honest and drugs (and their users) safe,” he told me. He started to report his findings to others on the forums, and then created a thread with summaries of the test results, as well as comments from the vendors if they provided it. Other users were soon basing their decisions on what to buy on elmo666‘s tests.

        “I’m defo trying the cola based on the incredibly helpful elmo and his energy control results and recommendations,” wrote user jayk1984. On top of this, elmo666 plans to launch an independent site on the deep web that will collate all of these results, which should act as a resource for users of all the marketplaces.

        As word of elmo666's efforts spread, he began getting requests from drug dealers who wanted him to use their wares for testing. Clearly, they figured that a positive result from Energy Control would be a fantastic marketing tool to draw more customers. They even offered elmo666 free samples. (He passed.)

        Meanwhile, some in the purchasing community are arguing that those running markets on the deep web should be providing quality control themselves. PlutoPete told me over the phone that he had been in discussions about this with Dread Pirate Roberts, the pseudonymous owner of the original Silk Road site. “We [had been] talking about that on a more organized basis on Silk Road 1, doing lots of anonymous buys to police each category. But of course they took the thing [Silk Road] down before we got it properly off the ground,” he lamented.

        But perhaps it is best that the users, those who are actually consuming the drugs, remain in charge of shaming dealers and warning each other. “It’s our responsibility to police the market based on reviews and feedback,” elmo666 wrote in an Evolution forum post. It seems that in the lawless space of the deep web, where everything from child porn to weapons are sold openly, users have cooperated in an organic display of self-regulation to stamp out those particular batches of drugs that are more likely to harm users.

        “That’s always been the case with the deep web,” PlutoPete told me. Indeed, ever since Silk Road, a stable of the drug markets has been the review system, where buyers can leave a rating and feedback for vendors, letting others know about the reliability of the seller. But DoctorX’s lab, rigorously testing the products with scientific instruments, takes it a step further.

        Photo by Joan Bardeletti

        “In the white market, they have quality control. In the dark market, it should be the same,” Cristina Gil Lladanosa says to me before I leave the Barcelona lab.

        A week after I visit the lab, the results of the MDMA arrive in my inbox: it is 85 percent pure, with no indications of other active ingredients. Whoever ordered that sample from the digital shelves of the deep web, and had it shipped to their doorstep in Canada, got hold of some seriously good, and relatively safe drugs. And now they know it.

        Top photo by Joan Bardeletti

        Follow Backchannel: Twitter |Facebook

        Go to Backchannel
        Published in
        Backchannel
        Mining the tech world for lively and meaningful tales and analysis.
        Email me when there are new stories in the publication
        Written on by
        Joseph Cox
        Encryption Enthusiast. Words for Motherboard, WIRED, Daily Dot. Documentaries for VICE. OTR chat: jfcox@jabber.ccc.de
        Thanks to Steven Levy.
        ================================================ FILE: test/test-pages/keep-tabular-data/expected-metadata.json ================================================ { "title": "Friday Facts #282 - 0.17 in sight | Factorio", "byline": null, "dir": null, "excerpt": "Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts", "siteName": "Factorio.com", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/keep-tabular-data/expected.html ================================================

        Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts

        The release plan (kovarex)

        This week was the time to close and finish all the things that will go to 0.17.0.

        Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.

        If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February.

        After release plan

        Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time.

        The smaller releases will contain mainly:

        • Final looks and behaviour of new GUI screens as they will be finished.
        • New graphics.
        • New sounds and sound system tweaks.
        • Mini tutorial additions and tweaks.

        This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates.

        The GUI progress (kovarex)

        There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional.

        General UX UX draft UX review UI mockup UI review Implementation draft Implementation review Final review
        Load map
        Save map
        Graphics settings
        Control settings
        Sound settings
        Interface settings
        Other settings
        Map generator
        Quick bar Twinsen
        Train GUI kovarex
        Technology GUI Oxyd
        Technology tooltip Oxyd
        Blueprint library kovarex
        Shortcut bar Oxyd
        Character screen Dominik
        Help overlay kovarex
        Manage/Install mods Rseding
        Recipe/item/Entity tooltip Twinsen
        Chat icon selector ?
        New game ?
        Menu structure ?
        Main screen chat ?
        Recipe explorer ?

        * Newly finished things since the last update in FFF-277.

        Blueprint library

        The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in FFF-278) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction.

        So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in FFF-255. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away.

        In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks:

        We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint.

        The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on.

        The hand

        Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case.

        This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest.


        The hand is protecting the slot from the robots.

        Terrain generation updates (TOGoS)

        Everyone has different opinions about what makes a good Factorio world. I've been working on several changes for 0.17, but the overarching theme has been to make the map generator options screen more intuitive and more powerful.

        This was talked about somewhat in an earlier FFF (FFF-258) regarding ore placement, but since then we found more stuff to fix.

        Biter Bases

        In 0.16, the size control for biter bases didn't have much effect. The frequency control changed the frequency, but that also decreased the size of bases, which wasn't generally what people wanted.

        For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base.


        New preview UI showing the effects of enemy base controls. In reality the preview takes a couple seconds to regenerate after every change, but the regeneration part is skipped in this animation to clearly show the effects of the controls.

        If you don't like the relatively uniform-across-the-world placement of biters, there are changes under the hood to make it easier for modders to do something different. Placement is now based on NamedNoiseExpressions "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity". By overriding any of those, a modder could easily create a map where biters are found only at high elevations, or only near water, or correlate enemy placement with that of resources, or any other thing that can be expressed as a function of location.

        Cliffs

        We've added a 'continuity' control for cliffs. If you really like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces. Or you can turn it way down to make cliffs very rare or be completely absent.


        Changing cliff frequency and continuity. Since cliffs are based on elevation, you'll have to turn frequency way up if you want lots of layers even near the starting lake.

        Biome Debugging

        Tile placement is based on a range of humidity and 'aux' values (humidity and aux being properties that vary at different points across the world) that are suitable for each type of tile. For example: grass is only placed in places with relatively high humidity, and desert (not to be confused with plain old sand) only gets placed where aux is high. We've taken to calling these constraints 'rectangles', because when you plot each tile's home turf on a chart of humidity and aux, they are shown as rectangles.

        It's hard to make sense of the rectangles just by looking at the autoplace code for each tile, so I wrote a script to chart them. This allowed us to ensure that they were arranged as we wanted, with no gaps between them, and with overlap in some cases.


        Rectangles.

        Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story, since tile placement also depends on a noise layer specific to each tile type, and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up). So to further help us visualize how humidity, aux, tile-specific noise, and autoplace controls worked together to determine tiles on the map, there are a couple of alternate humidity and aux generators that simply vary them linearly from north-south and west-east, respectively.


        Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively.

        This map helped us realize that, rather than having controls for each different type of tile, it made more sense to just control moisture and aux (which is called 'terrain type' in the GUI, because 'aux' doesn't mean anything).


        Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty.

        A pet project of mine has been to put controls in the map generator GUI so that we could select generators for various tile properties (temperature, aux, humidity, elevation, etc) at map-creation time without necessarily needing to involve mods. This was useful for debugging the biome rectangles, but my ulterior motive was to, at least in cases where there are multiple options, show the generator information to players. A couple of reasons for this:

        • It was already possible for mods to override tile property generators via presets, but we didn't have a place to show that information in the UI. So switching between presets could change hidden variables in non-obvious ways and lead to a lot of confusion.
        • I had dreams of shipping alternate elevation generators in the base game.

        Water Placement

        For 0.16 I attempted to make the shape of continents more interesting. Some people really liked the new terrain, or at least managed to find some settings that made it work for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days.

        So in 0.17 we're restoring the default elevation generator to one very similar to that used by 0.12. Which means large, sometimes-connected lakes.

        The water 'frequency' control was confusing to a lot of people including myself. It could be interpreted as "how much water", when the actual effect was to inversely scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water. So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage', which do the same thing, but in a hopefully more obvious way. Larger scale means larger land features, while more coverage means more of the map covered in water.

        New Map Types

        In order to ensure a decent starting area, the elevation generator always makes a plateau there (so you'll never spawn in the middle of the ocean), and a lake (so you can get a power plant running). Depending on what's going on outside of that plateau, this sometimes resulted in a circular ring of cliffs around the starting point, which looked very artificial, and we wanted to reduce that effect.

        In the process of solving that problem I created another custom generator for debugging purposes. This one simply generated that starting area plateau in an endless ocean. I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it to illustrate the mechanics behind generating the starting area.

        The rest of the team liked that setting so much that we're making it a player-selectable option. So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12, and 'Island', which gives you a single large-ish island at the starting point. There's a slider to let you change the size of the island(s).

        Maps with multiple starting points will have multiple islands.


        PvP islands!

        And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings) to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options, but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts (e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider, rather than being just vaguely related to it somehow), we're switching to storing them as numbers. This has the side-effect that if you don't mind editing some JSON, you'll be able to create maps with values outside the range provided by the GUI sliders.

        Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum.

        High-res accumulators (Ernestas, Albert)

        The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was.

        The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same.

        As always, let us know what you think on our forum.

        ================================================ FILE: test/test-pages/keep-tabular-data/source.html ================================================ Friday Facts #282 - 0.17 in sight | Factorio
        log in sign up

        Friday Facts #282 - 0.17 in sight

        Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts

        The release plan (kovarex)

        This week was the time to close and finish all the things that will go to 0.17.0.

        Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.

        If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February.

        After release plan

        Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time.

        The smaller releases will contain mainly:

        • Final looks and behaviour of new GUI screens as they will be finished.
        • New graphics.
        • New sounds and sound system tweaks.
        • Mini tutorial additions and tweaks.

        This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates.

        The GUI progress (kovarex)

        There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional.

        General UX UX draft UX review UI mockup UI review Implementation draft Implementation review Final review
        Load map
        Save map
        Graphics settings
        Control settings
        Sound settings
        Interface settings
        Other settings
        Map generator
        Quick bar Twinsen
        Train GUI kovarex
        Technology GUI Oxyd
        Technology tooltip Oxyd
        Blueprint library kovarex
        Shortcut bar Oxyd
        Character screen Dominik
        Help overlay kovarex
        Manage/Install mods Rseding
        Recipe/item/Entity tooltip Twinsen
        Chat icon selector ?
        New game ?
        Menu structure ?
        Main screen chat ?
        Recipe explorer ?
        * Newly finished things since the last update in FFF-277.

        Blueprint library

        The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in FFF-278) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction.

        So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in FFF-255. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away.

        In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks:

        We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint.

        The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on.

        The hand

        Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case.

        This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest.


        The hand is protecting the slot from the robots.

        Terrain generation updates (TOGoS)

        Everyone has different opinions about what makes a good Factorio world. I've been working on several changes for 0.17, but the overarching theme has been to make the map generator options screen more intuitive and more powerful.

        This was talked about somewhat in an earlier FFF (FFF-258) regarding ore placement, but since then we found more stuff to fix.

        Biter Bases

        In 0.16, the size control for biter bases didn't have much effect. The frequency control changed the frequency, but that also decreased the size of bases, which wasn't generally what people wanted.

        For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base.


        New preview UI showing the effects of enemy base controls. In reality the preview takes a couple seconds to regenerate after every change, but the regeneration part is skipped in this animation to clearly show the effects of the controls.

        If you don't like the relatively uniform-across-the-world placement of biters, there are changes under the hood to make it easier for modders to do something different. Placement is now based on NamedNoiseExpressions "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity". By overriding any of those, a modder could easily create a map where biters are found only at high elevations, or only near water, or correlate enemy placement with that of resources, or any other thing that can be expressed as a function of location.

        Cliffs

        We've added a 'continuity' control for cliffs. If you really like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces. Or you can turn it way down to make cliffs very rare or be completely absent.


        Changing cliff frequency and continuity. Since cliffs are based on elevation, you'll have to turn frequency way up if you want lots of layers even near the starting lake.

        Biome Debugging

        Tile placement is based on a range of humidity and 'aux' values (humidity and aux being properties that vary at different points across the world) that are suitable for each type of tile. For example: grass is only placed in places with relatively high humidity, and desert (not to be confused with plain old sand) only gets placed where aux is high. We've taken to calling these constraints 'rectangles', because when you plot each tile's home turf on a chart of humidity and aux, they are shown as rectangles.

        It's hard to make sense of the rectangles just by looking at the autoplace code for each tile, so I wrote a script to chart them. This allowed us to ensure that they were arranged as we wanted, with no gaps between them, and with overlap in some cases.


        Rectangles.

        Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story, since tile placement also depends on a noise layer specific to each tile type, and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up). So to further help us visualize how humidity, aux, tile-specific noise, and autoplace controls worked together to determine tiles on the map, there are a couple of alternate humidity and aux generators that simply vary them linearly from north-south and west-east, respectively.


        Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively.

        This map helped us realize that, rather than having controls for each different type of tile, it made more sense to just control moisture and aux (which is called 'terrain type' in the GUI, because 'aux' doesn't mean anything).


        Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty.

        A pet project of mine has been to put controls in the map generator GUI so that we could select generators for various tile properties (temperature, aux, humidity, elevation, etc) at map-creation time without necessarily needing to involve mods. This was useful for debugging the biome rectangles, but my ulterior motive was to, at least in cases where there are multiple options, show the generator information to players. A couple of reasons for this:

        • It was already possible for mods to override tile property generators via presets, but we didn't have a place to show that information in the UI. So switching between presets could change hidden variables in non-obvious ways and lead to a lot of confusion.
        • I had dreams of shipping alternate elevation generators in the base game.

        Water Placement

        For 0.16 I attempted to make the shape of continents more interesting. Some people really liked the new terrain, or at least managed to find some settings that made it work for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days.

        So in 0.17 we're restoring the default elevation generator to one very similar to that used by 0.12. Which means large, sometimes-connected lakes.

        The water 'frequency' control was confusing to a lot of people including myself. It could be interpreted as "how much water", when the actual effect was to inversely scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water. So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage', which do the same thing, but in a hopefully more obvious way. Larger scale means larger land features, while more coverage means more of the map covered in water.

        New Map Types

        In order to ensure a decent starting area, the elevation generator always makes a plateau there (so you'll never spawn in the middle of the ocean), and a lake (so you can get a power plant running). Depending on what's going on outside of that plateau, this sometimes resulted in a circular ring of cliffs around the starting point, which looked very artificial, and we wanted to reduce that effect.

        In the process of solving that problem I created another custom generator for debugging purposes. This one simply generated that starting area plateau in an endless ocean. I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it to illustrate the mechanics behind generating the starting area.

        The rest of the team liked that setting so much that we're making it a player-selectable option. So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12, and 'Island', which gives you a single large-ish island at the starting point. There's a slider to let you change the size of the island(s).

        Maps with multiple starting points will have multiple islands.


        PvP islands!

        And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings) to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options, but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts (e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider, rather than being just vaguely related to it somehow), we're switching to storing them as numbers. This has the side-effect that if you don't mind editing some JSON, you'll be able to create maps with values outside the range provided by the GUI sliders.

        Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum.

        High-res accumulators (Ernestas, Albert)

        The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was.

        The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same.

        As always, let us know what you think on our forum.

        Copyright © 2015 - 2019 Wube Software - all rights reserved.
        Terms of Service | Privacy | Imprint | Presskit | FAQ | RSS | Jobs
        ================================================ FILE: test/test-pages/la-nacion/expected-metadata.json ================================================ { "title": "Una solución no violenta para la cuestión mapuche", "byline": null, "dir": null, "excerpt": "Los pueblos indígenas reclaman por derechos que permanecen incumplidos, por eso es más eficiente canalizar la protesta que reprimirla", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/la-nacion/expected.html ================================================

        Los pueblos indígenas reclaman por derechos que permanecen incumplidos, por eso es más eficiente canalizar la protesta que reprimirla

        Abdullah Ocalan, el líder independentista kurdo, desembarcó en Italia en noviembre de 1998 y pidió asilo político. Arrastraba un pedido de captura de Turquía, donde era acusado por terrorismo. El ex comunista Massimo D'Alema, recién asumido, dudaba. Acoger a Ocalan implicaba comprarse un problema con un aliado de la OTAN e importar un conflicto ajeno, pero deportarlo lo exponía a la pena de muerte, legal en Turquía pero inadmisible en la Unión Europea. Optó por la estrecha avenida del medio: se ignoró el mandato de captura al tiempo que se negó el asilo, presionando a Ocalan para que se fuera por las suyas. Tras una carambola a tres bandas, fue capturado por agentes turcos en Kenia, donde se encontraba bajo la protección del embajador griego, mientras intentaba abordar un avión hacia Holanda. Desde febrero de 1999 permanece en una cárcel de máxima seguridad en la isla turca de Imrali.

        Uno de los autores de esta columna vivía en Italia en esa época y siguió la crisis de cerca; el otro la estudió en profundidad, años más tarde. Pero no hacía falta: cualquiera puede encontrar esta información a un clic de distancia. Eso fue lo que no hizo un periodista de un diario argentino, que no es la nacion. La semana pasada se publicaron extractos de un "informe de carácter secreto" que mencionaba supuestos contactos internacionales de organizaciones mapuches. Entre ellos aparecía Ocalan, a quien el informe ubicó "con domicilios en Palermo y en el centro porteño", y aseguraba incluso que había sido visto "en Neuquén, Río Negro y Chubut durante el juicio a Jones Huala".

        Foto: LA NACION

        Esta falsa noticia fue la más rocambolesca de una larga cadena. Dos hechos quedaron en evidencia: primero, que hay periodistas que no chequean la información; segundo, que los servicios de inteligencia los utilizan para manipular la agenda pública. Y sobre los servicios hay dos posibilidades: o son burros o son perversos. Las opciones no son excluyentes, aunque cualquiera alcanza para tornarlos indignos de confianza. Sin embargo, de ellos proviene la información que alimenta a muchos medios de comunicación y, aún más grave, al Estado argentino.

        El reguero de noticias falsas y vínculos brumosos tiene, paradójicamente, un objetivo prístino: asociar la acción de los grupos mapuches con el terrorismo internacional. Comunicadores, analistas y escritores alineados con el discurso oficial llegaron a relacionar las ideas de las organizaciones patagónicas con las de Estado Islámico (ISIS) de Irak y Siria. El terrorismo carece de definiciones consensuales y ha sido utilizado para emparentar cosas bien diferentes. Aunque el líder mapuche más radicalizado (y menos representativo) declare que propician "un proceso de construcción de autonomía sin pedirle permiso al Estado", vincular a un grupo que reclama tierras en la región de sus ancestros con otro que busca gobernar el mundo según sus normas religiosas y ha masacrado a miles de personas requiere de una operación intelectual tan audaz como inadecuada.

        La asociación con el movimiento kurdo, en cambio, asoma menos inverosímil. Desde su arresto, Ocalan transformó su pensamiento: de una visión nacionalista con inspiración estalinista evolucionó al confederalismo democrático, una propuesta de organización comunal, ecologista, más apegada a las raíces locales que a las fronteras nacionales. Parece lógico que esas ideas resuenen en agrupamientos indígenas, que reivindican una organización anterior a la consolidación de los Estados sudamericanos. Los paralelos, sin embargo, terminan allí. En Chile, donde el conflicto ha tenido su desarrollo más dramático, la Sociedad de Fomento Agrícola denunció en 2014 que los insurrectos causaron daños por 10 millones de dólares y la muerte de tres agricultores y un carabinero a lo largo de 15 años; en la Argentina, por ahora, se registran actos de vandalismo, ocupaciones de tierras y cortes de rutas aislados. En contraste, el conflicto entre el Partido de los Trabajadores del Kurdistán y la República de Turquía se cobró cerca de 40.000 vidas en los años 90 y lleva más de 2000 desde la reanudación de hostilidades en 2015.

        Consultada sobre esta desproporción, una fuente de los servicios nos la resumió así: "La estrategia de la Coordinadora Arauco-Malleco (CAM), de Chile, y ahora de la Resistencia Ancestral Mapuche (RAM), más que matar directamente, es realizar sabotajes, movilizaciones, ataques a iglesias y empresas y mucha prensa". ¡En Medio Oriente pagarían por un terrorismo así! Ningún hecho de violencia debe ser minimizado, pero las analogías no resisten prueba.

        La "cuestión mapuche" es social antes que policial. La Constitución manda "reconocer la preexistencia étnica y cultural de los pueblos indígenas argentinos. Garantizar el respeto a su identidad?; reconocer la personería jurídica de sus comunidades, y la posesión y propiedad comunitarias de las tierras que tradicionalmente ocupan; y regular la entrega de otras aptas y suficientes para el desarrollo humano". Estos derechos permanecen incumplidos. Y no son un capricho chavista: los países que reputamos serios también los reconocen. En Estados Unidos, las reservaciones indígenas ocupan 80.000 kilómetros cuadrados, el 1,3% de la superficie del país (y 400 veces la superficie de la ciudad de Buenos Aires). En Canadá, unas 2300 reservas ocupan 28.000 kilómetros cuadrados. Australia otorga a los pueblos indígenas más de la mitad de los territorios del norte del país y son los nativos quienes negocian con las empresas mineras los permisos para que operen en sus tierras. En Nueva Zelanda existen tribunales especiales con jurisdicción sobre las tierras ancestrales de los maoríes; una de sus ventajas es que empoderan a los aborígenes individualmente, liberándolos del yugo de los caciques.

        La protesta social es indisociable de la democracia. Cuando desborda, recanalizarla es más eficiente que reprimirla: ahí reside el arte del acuerdo. En la Argentina la tarea es delicada porque pocos confían en la imparcialidad de las instituciones. Entonces, cada actor reivindica sus intereses con los medios de que dispone: los sindicatos hacen huelga, los estudiantes toman colegios, los empresarios cierran las fábricas y todos hacen piquetes. El politólogo Samuel Huntington definía una sociedad así como pretoriana y el jurista Carlos Nino llamó a la Argentina "un país al margen de la ley". Al movilizarse por sus derechos y desconfiar del Estado, la comunidad mapuche se demuestra bien argentina.

        Las cinco provincias patagónicas tienen una población similar a la de La Matanza. A diferencia de los Estados Unidos, que se integraron hacia el oeste otorgando parcelas de tierra a los colonizadores, y de Brasil, donde el rol de ocupación y desarrollo territorial fue cumplido por las fuerzas armadas, la Argentina obvió la tarea integradora tras consolidar su soberanía a finales del siglo XX. Hoy sobra tierra y falta gente. Gobernar sigue siendo poblar, pero también integrar.

        Seamos claros: ningún individuo u organización tiene derecho a violar la ley. Pero el problema histórico del Estado argentino no fue tanto quiénes lo desafiaron como quiénes lo gobernaron. Cambiemos.

        Andrés Malamud es politólogo e investigador en la Universidad de Lisboa. Martín Schapiro es abogado administrativista y analista internacional

        ================================================ FILE: test/test-pages/la-nacion/source.html ================================================  Una solución no violenta para la cuestión mapuche - 07.12.2017 - LA NACION Si usted es una persona con dificultades visuales, navegue el sitio desde aquí

        Una solución no violenta para la cuestión mapuche

        Los pueblos indígenas reclaman por derechos que permanecen incumplidos, por eso es más eficiente canalizar la protesta que reprimirla

        0

        Abdullah Ocalan, el líder independentista kurdo, desembarcó en Italia en noviembre de 1998 y pidió asilo político. Arrastraba un pedido de captura de Turquía, donde era acusado por terrorismo. El ex comunista Massimo D'Alema, recién asumido, dudaba. Acoger a Ocalan implicaba comprarse un problema con un aliado de la OTAN e importar un conflicto ajeno, pero deportarlo lo exponía a la pena de muerte, legal en Turquía pero inadmisible en la Unión Europea. Optó por la estrecha avenida del medio: se ignoró el mandato de captura al tiempo que se negó el asilo, presionando a Ocalan para que se fuera por las suyas. Tras una carambola a tres bandas, fue capturado por agentes turcos en Kenia, donde se encontraba bajo la protección del embajador griego, mientras intentaba abordar un avión hacia Holanda. Desde febrero de 1999 permanece en una cárcel de máxima seguridad en la isla turca de Imrali.

        Uno de los autores de esta columna vivía en Italia en esa época y siguió la crisis de cerca; el otro la estudió en profundidad, años más tarde. Pero no hacía falta: cualquiera puede encontrar esta información a un clic de distancia. Eso fue lo que no hizo un periodista de un diario argentino, que no es la nacion. La semana pasada se publicaron extractos de un "informe de carácter secreto" que mencionaba supuestos contactos internacionales de organizaciones mapuches. Entre ellos aparecía Ocalan, a quien el informe ubicó "con domicilios en Palermo y en el centro porteño", y aseguraba incluso que había sido visto "en Neuquén, Río Negro y Chubut durante el juicio a Jones Huala".

        Foto: LA NACION

        Esta falsa noticia fue la más rocambolesca de una larga cadena. Dos hechos quedaron en evidencia: primero, que hay periodistas que no chequean la información; segundo, que los servicios de inteligencia los utilizan para manipular la agenda pública. Y sobre los servicios hay dos posibilidades: o son burros o son perversos. Las opciones no son excluyentes, aunque cualquiera alcanza para tornarlos indignos de confianza. Sin embargo, de ellos proviene la información que alimenta a muchos medios de comunicación y, aún más grave, al Estado argentino.

        El reguero de noticias falsas y vínculos brumosos tiene, paradójicamente, un objetivo prístino: asociar la acción de los grupos mapuches con el terrorismo internacional. Comunicadores, analistas y escritores alineados con el discurso oficial llegaron a relacionar las ideas de las organizaciones patagónicas con las de Estado Islámico (ISIS) de Irak y Siria. El terrorismo carece de definiciones consensuales y ha sido utilizado para emparentar cosas bien diferentes. Aunque el líder mapuche más radicalizado (y menos representativo) declare que propician "un proceso de construcción de autonomía sin pedirle permiso al Estado", vincular a un grupo que reclama tierras en la región de sus ancestros con otro que busca gobernar el mundo según sus normas religiosas y ha masacrado a miles de personas requiere de una operación intelectual tan audaz como inadecuada.

        La asociación con el movimiento kurdo, en cambio, asoma menos inverosímil. Desde su arresto, Ocalan transformó su pensamiento: de una visión nacionalista con inspiración estalinista evolucionó al confederalismo democrático, una propuesta de organización comunal, ecologista, más apegada a las raíces locales que a las fronteras nacionales. Parece lógico que esas ideas resuenen en agrupamientos indígenas, que reivindican una organización anterior a la consolidación de los Estados sudamericanos. Los paralelos, sin embargo, terminan allí. En Chile, donde el conflicto ha tenido su desarrollo más dramático, la Sociedad de Fomento Agrícola denunció en 2014 que los insurrectos causaron daños por 10 millones de dólares y la muerte de tres agricultores y un carabinero a lo largo de 15 años; en la Argentina, por ahora, se registran actos de vandalismo, ocupaciones de tierras y cortes de rutas aislados. En contraste, el conflicto entre el Partido de los Trabajadores del Kurdistán y la República de Turquía se cobró cerca de 40.000 vidas en los años 90 y lleva más de 2000 desde la reanudación de hostilidades en 2015.

        Consultada sobre esta desproporción, una fuente de los servicios nos la resumió así: "La estrategia de la Coordinadora Arauco-Malleco (CAM), de Chile, y ahora de la Resistencia Ancestral Mapuche (RAM), más que matar directamente, es realizar sabotajes, movilizaciones, ataques a iglesias y empresas y mucha prensa". ¡En Medio Oriente pagarían por un terrorismo así! Ningún hecho de violencia debe ser minimizado, pero las analogías no resisten prueba.

        La "cuestión mapuche" es social antes que policial. La Constitución manda "reconocer la preexistencia étnica y cultural de los pueblos indígenas argentinos. Garantizar el respeto a su identidad?; reconocer la personería jurídica de sus comunidades, y la posesión y propiedad comunitarias de las tierras que tradicionalmente ocupan; y regular la entrega de otras aptas y suficientes para el desarrollo humano". Estos derechos permanecen incumplidos. Y no son un capricho chavista: los países que reputamos serios también los reconocen. En Estados Unidos, las reservaciones indígenas ocupan 80.000 kilómetros cuadrados, el 1,3% de la superficie del país (y 400 veces la superficie de la ciudad de Buenos Aires). En Canadá, unas 2300 reservas ocupan 28.000 kilómetros cuadrados. Australia otorga a los pueblos indígenas más de la mitad de los territorios del norte del país y son los nativos quienes negocian con las empresas mineras los permisos para que operen en sus tierras. En Nueva Zelanda existen tribunales especiales con jurisdicción sobre las tierras ancestrales de los maoríes; una de sus ventajas es que empoderan a los aborígenes individualmente, liberándolos del yugo de los caciques.

        La protesta social es indisociable de la democracia. Cuando desborda, recanalizarla es más eficiente que reprimirla: ahí reside el arte del acuerdo. En la Argentina la tarea es delicada porque pocos confían en la imparcialidad de las instituciones. Entonces, cada actor reivindica sus intereses con los medios de que dispone: los sindicatos hacen huelga, los estudiantes toman colegios, los empresarios cierran las fábricas y todos hacen piquetes. El politólogo Samuel Huntington definía una sociedad así como pretoriana y el jurista Carlos Nino llamó a la Argentina "un país al margen de la ley". Al movilizarse por sus derechos y desconfiar del Estado, la comunidad mapuche se demuestra bien argentina.

        Las cinco provincias patagónicas tienen una población similar a la de La Matanza. A diferencia de los Estados Unidos, que se integraron hacia el oeste otorgando parcelas de tierra a los colonizadores, y de Brasil, donde el rol de ocupación y desarrollo territorial fue cumplido por las fuerzas armadas, la Argentina obvió la tarea integradora tras consolidar su soberanía a finales del siglo XX. Hoy sobra tierra y falta gente. Gobernar sigue siendo poblar, pero también integrar.

        Seamos claros: ningún individuo u organización tiene derecho a violar la ley. Pero el problema histórico del Estado argentino no fue tanto quiénes lo desafiaron como quiénes lo gobernaron. Cambiemos.

        Andrés Malamud es politólogo e investigador en la Universidad de Lisboa. Martín Schapiro es abogado administrativista y analista internacional

        Enviá tu comentario

        Los comentarios publicados son de exclusiva responsabilidad de sus autores y las consecuencias derivadas de ellos pueden ser pasibles de sanciones legales. Aquel usuario que incluya en sus mensajes algún comentario violatorio del reglamento será eliminado e inhabilitado para volver a comentar. Enviar un comentario implica la aceptación del Reglamento.

        Para poder comentar tenés que ingresar con tu usuario de LA NACION.
        ================================================ FILE: test/test-pages/lazy-image-1/expected-metadata.json ================================================ { "title": "Node.js and CPU profiling on production (in real-time without downtime)", "byline": "Vincent Vallet", "dir": null, "lang": "en", "excerpt": "How to run a CPU profiling with Node.js on your production in real-time and without interruption of service.", "siteName": "Voodoo Engineering", "publishedTime": "2019-10-18T17:23:34.816Z", "readerable": true } ================================================ FILE: test/test-pages/lazy-image-1/expected.html ================================================

        Vincent Vallet

        Why CPU monitoring is important?

        I work at Voodoo, a French company that creates mobile video games. We have a lot of challenges with performance, availability, and scalability because of the insane amount of traffic our infrastructure supports (billions of events/requests per day …… no joke!). In this setting, every metric is important and gives us a lot of information about the state of our system.

        When working with Node.js one of the most critical resources to monitor is the CPU. Most of the time, when working on a low traffic API or project we don’t realize how many simple lines of code can have a huge impact on CPU. On the other hand, when traffic increases, a simple mistake can cost dearly.

        Resources

        What kind of resources does your application need? In most cases, we focus on memory and CPU. Good monitoring of these two elements is mandatory for an application running on production.

        For memory, constant monitoring is the best practice to track the worst developer nightmare a.k.a memory leak.

        Memory leak in action

        A good way to debug memory leak is a memory dump and/or memory sampling but this is not the subject.

        (for more details about V8 and its garbage collector you can read my previous article here)

        Stay focused on the CPU!

        Most of the time we monitor this resource with a simple solution allowing us to get a graph representing CPU consumption over time. If we want to be reactive we add an alarm, based on a threshold, to warn us when CPU usage is too high.

        Basic CPU monitoring

        And what next? We don’t have data about the state of the instance when the CPU usage has increased. So we can’t determine why we had this peak, at least not without an important time of debugging, comparing log, etc. This is exactly why you need to use CPU profiling.

        CPU profiling: what’s the difference with CPU monitoring?

        “Most commonly, profiling information serves to aid program optimization. Profiling is achieved by instrumenting either the program source code or its binary executable form using a tool called a profiler”

        Basically, for Node.js, CPU profiling is nothing more than collecting data about functions which are CPU consuming. And ideally, get a graphic representation of the collected data a.k.a “flame graph” or “flame chart”.

        It will help you to track the exact file, line, and function which takes the most time to execute.

        What about existing solutions?

        Add arguments to Node.js

        Node.js provides a way to collect data about CPU with two command lines.

        The first command just executes your application, the argument just tells to V8 engine to collect data. When you stop your script all information is stored in a file. 

        node --prof app.js

        Output of — prof

        It is not very clear, is it?

        That’s why you just need to run this second command to transform your raw file into a more human-readable output. 

        node --prof-process isolate-0xnnnnn-v8.log > processed.txt

        The output of — prof-process

        It seems better, here you can determine which function consumes the most of CPU (percentage of the time).

        ClinicJs

        ClinicJs is a set of tools that allow you to collect data and display performance charts. With “clinic flame” you can generate a flame graph based on CPU consumption.

        Flame chart

        But once again, you have to stop your app, launch the tool, then terminate the script in order to display the graph (files are generated on the disk).

        For more details, you can see the project.

        To sum up, here is the list of drawbacks of the two previous solutions.

        • Downtime (you should kill your application to collect the data)
        • Performance overhead
        • Data collected locally
        • Need external tools (ClinicJs)

        In conclusion: these are good solutions to debug on development environments and/or on a local machine.

        Unfortunately, CPU issues have a worrying tendency to occur on production, and when you are not in front of your screen.

        Inspector

        “Inspector” refers to an API thanks to which you can debug your application. By debugging we mean to be able to connect directly to the core of Node.js to collect real-time data about the process.

        A module, available since version 8.x of Node.js, provides this kind of feature. There are two advantages to use it:

        • it’s native (no additional installation required)
        • it can be used programmatically (no interruption)

        And here is how to make a CPU profiling with this module:

        As you can see, all the data is returned in variable “profile”. Basically, it’s a simple JSON object representing all the call stack and the CPU consumption for each function. And if you want to use an Async/await syntax you can install the module “inspector-api”. 

        npm install inspector-api --save

        It also comes with a built-in exporter to send data to S3, with this method you don’t write anything on the disk!

        If you use another storage system you can just collect the data and export it by yourself.

        And now, CPU profiling on-demand!

        We have an API that we want to test with autocannon tool. At this step, our project is able to serve around 200 requests in 20 seconds. There is probably a mistake somewhere in the code which slows down our application.

        But now, what if we want to trigger a CPU profiling remotely (without ssh connection to the server)? It’s possible using Websocket, SSE or any other technology to send a message to your instance.

        Here is a simple example of a server using the “ws” module to send a message to a unique instance.

        Of course, it only works with one instance, but it’s a fake project to demonstrate the principle ;)

        Now we can request our server to ask it to send a message to our instance and start/stop a CPU profiling. In your instance, you can handle the CPU profiling like this:

        To sum up: we are able to trigger a CPU profiling, on-demand, in real-time, without interruption or connection to the server. Data can be collected on the disk (and extracted later) or can be sent to S3 (or any other system, PR are welcomed on the inspector-api project).

        And because the profiler is a part of V8 itself, the format of the generated JSON file is compatible with the Chrome dev tools.

        How can we identify an issue?

        A CPU profiling should be read like this:

        • the x-axis shows the stack profile population
        • the y-axis shows stack depth

        What does it mean?

        The larger is a box (a function call) the more it consumed CPU. So a good CPU profiling should look like a “flame” graph where each stack is the finest possible.

        In our example, every request try to generate a token. For this purpose, it calls the function pbkdf2 which is CPU consuming. Our CPU profile looks like a sequence of big blocks of time, like if the last function in the call stack takes 99% of the total time.

        The CPU profiling after optimizations, with the same time range.

        CPU profiling after optimizations

        As you can notice, we have to zoom to the profile if we want to see the call stack, because after optimizations the API was able to take a lot more traffic. Now every function in the call stack looks like a microtask.

        And now our application is able to serve more than 200,000 requests in 20 seconds; we increased the performance by a factor of 100k!

        More than just CPU profiling

        With the inspector module, you can do much more than just CPU profiling, here is a non-exhaustive list:

        • memory dump & memory sampling
        • code coverage
        • use the debugger in real-time

        Warnings

        Every tool, even the most powerful, comes with its own disadvantages. If you enable the profiler and/or the debugger on your production you have to keep an eye on two things:

        1) performance overhead

        A profiler needs to use CPU to work and it collects data into memory. The longer you let it run and the more CPU / memory it will need. This is why you should begin with very short CPU profiling, no more than a few seconds between the start and stop command. And never forget to monitor the impact of the profiler on your own infrastructure. If everything is fine you can increase the time and the frequency of CPU profiling.

        One more very important thing: never forget to always stop a started CPU profiling. You can add a timer to automatically call the stop function after a while.

        2) security

        Using the inspector in Node.js it’s like opening the door of the core of your application. You should be very careful about who can use features like CPU profiling and/or the debugger. Never make the inspector “public” as being able to launch a feature from an unsafe route (not protected with an authentification mechanism). Even the collected data can be seen as critical, never send it to a system you do not trust.

        Conclusion

        CPU profiling is really a must-have tool for every developer. And now, with some precautions, we can run it on production thanks to the amazing work done by the V8 and Node.js team.

        The inspector module offers a lot more features than you can use to debug your application.

        I will write another article about using CPU profiling and the inspector on production on a high traffic project.

        Sources & links

        ================================================ FILE: test/test-pages/lazy-image-1/source.html ================================================ Node.js and CPU profiling on production (in real-time without downtime)

        Node.js and CPU profiling on production (in real-time without downtime)

        Vincent Vallet
        Oct 18, 2019 · 8 min read

        Why CPU monitoring is important?

        I work at Voodoo, a French company that creates mobile video games. We have a lot of challenges with performance, availability, and scalability because of the insane amount of traffic our infrastructure supports (billions of events/requests per day …… no joke!). In this setting, every metric is important and gives us a lot of information about the state of our system.

        When working with Node.js one of the most critical resources to monitor is the CPU. Most of the time, when working on a low traffic API or project we don’t realize how many simple lines of code can have a huge impact on CPU. On the other hand, when traffic increases, a simple mistake can cost dearly.

        Resources

        What kind of resources does your application need? In most cases, we focus on memory and CPU. Good monitoring of these two elements is mandatory for an application running on production.

        For memory, constant monitoring is the best practice to track the worst developer nightmare a.k.a memory leak.

        Memory leak in action

        A good way to debug memory leak is a memory dump and/or memory sampling but this is not the subject.

        (for more details about V8 and its garbage collector you can read my previous article here)

        Stay focused on the CPU!

        Most of the time we monitor this resource with a simple solution allowing us to get a graph representing CPU consumption over time. If we want to be reactive we add an alarm, based on a threshold, to warn us when CPU usage is too high.

        Basic CPU monitoring

        And what next? We don’t have data about the state of the instance when the CPU usage has increased. So we can’t determine why we had this peak, at least not without an important time of debugging, comparing log, etc. This is exactly why you need to use CPU profiling.

        CPU profiling: what’s the difference with CPU monitoring?

        “Most commonly, profiling information serves to aid program optimization. Profiling is achieved by instrumenting either the program source code or its binary executable form using a tool called a profiler”

        Basically, for Node.js, CPU profiling is nothing more than collecting data about functions which are CPU consuming. And ideally, get a graphic representation of the collected data a.k.a “flame graph” or “flame chart”.

        It will help you to track the exact file, line, and function which takes the most time to execute.

        What about existing solutions?

        Node.js provides a way to collect data about CPU with two command lines.

        The first command just executes your application, the argument just tells to V8 engine to collect data. When you stop your script all information is stored in a file. 

        node --prof app.js
        Output of — prof

        It is not very clear, is it?

        That’s why you just need to run this second command to transform your raw file into a more human-readable output. 

        node --prof-process isolate-0xnnnnn-v8.log > processed.txt
        The output of — prof-process

        It seems better, here you can determine which function consumes the most of CPU (percentage of the time).

        ClinicJs is a set of tools that allow you to collect data and display performance charts. With “clinic flame” you can generate a flame graph based on CPU consumption.

        Flame chart

        But once again, you have to stop your app, launch the tool, then terminate the script in order to display the graph (files are generated on the disk).

        For more details, you can see the project.

        To sum up, here is the list of drawbacks of the two previous solutions.

        • Downtime (you should kill your application to collect the data)
        • Performance overhead
        • Data collected locally
        • Need external tools (ClinicJs)

        In conclusion: these are good solutions to debug on development environments and/or on a local machine.

        Unfortunately, CPU issues have a worrying tendency to occur on production, and when you are not in front of your screen.

        Inspector

        “Inspector” refers to an API thanks to which you can debug your application. By debugging we mean to be able to connect directly to the core of Node.js to collect real-time data about the process.

        A module, available since version 8.x of Node.js, provides this kind of feature. There are two advantages to use it:

        • it’s native (no additional installation required)
        • it can be used programmatically (no interruption)

        And here is how to make a CPU profiling with this module:

        As you can see, all the data is returned in variable “profile”. Basically, it’s a simple JSON object representing all the call stack and the CPU consumption for each function. And if you want to use an Async/await syntax you can install the module “inspector-api”. 

        npm install inspector-api --save

        It also comes with a built-in exporter to send data to S3, with this method you don’t write anything on the disk!

        If you use another storage system you can just collect the data and export it by yourself.

        And now, CPU profiling on-demand!

        We have an API that we want to test with autocannon tool. At this step, our project is able to serve around 200 requests in 20 seconds. There is probably a mistake somewhere in the code which slows down our application.

        But now, what if we want to trigger a CPU profiling remotely (without ssh connection to the server)? It’s possible using Websocket, SSE or any other technology to send a message to your instance.

        Here is a simple example of a server using the “ws” module to send a message to a unique instance.

        Of course, it only works with one instance, but it’s a fake project to demonstrate the principle ;)

        Now we can request our server to ask it to send a message to our instance and start/stop a CPU profiling. In your instance, you can handle the CPU profiling like this:

        To sum up: we are able to trigger a CPU profiling, on-demand, in real-time, without interruption or connection to the server. Data can be collected on the disk (and extracted later) or can be sent to S3 (or any other system, PR are welcomed on the inspector-api project).

        And because the profiler is a part of V8 itself, the format of the generated JSON file is compatible with the Chrome dev tools.

        CPU profiling before optimization
        CPU profiling before optimization
        CPU profiling before optimization

        How can we identify an issue?

        A CPU profiling should be read like this:

        • the x-axis shows the stack profile population
        • the y-axis shows stack depth

        What does it mean?

        The larger is a box (a function call) the more it consumed CPU. So a good CPU profiling should look like a “flame” graph where each stack is the finest possible.

        In our example, every request try to generate a token. For this purpose, it calls the function pbkdf2 which is CPU consuming. Our CPU profile looks like a sequence of big blocks of time, like if the last function in the call stack takes 99% of the total time.

        The CPU profiling after optimizations, with the same time range.

        CPU profiling after optimizations

        As you can notice, we have to zoom to the profile if we want to see the call stack, because after optimizations the API was able to take a lot more traffic. Now every function in the call stack looks like a microtask.

        CPU profiling after optimization
        CPU profiling after optimization
        Zoom in the CPU profiling

        And now our application is able to serve more than 200,000 requests in 20 seconds; we increased the performance by a factor of 100k!

        More than just CPU profiling

        With the inspector module, you can do much more than just CPU profiling, here is a non-exhaustive list:

        • memory dump & memory sampling
        • code coverage
        • use the debugger in real-time

        Warnings

        Every tool, even the most powerful, comes with its own disadvantages. If you enable the profiler and/or the debugger on your production you have to keep an eye on two things:

        1) performance overhead

        A profiler needs to use CPU to work and it collects data into memory. The longer you let it run and the more CPU / memory it will need. This is why you should begin with very short CPU profiling, no more than a few seconds between the start and stop command. And never forget to monitor the impact of the profiler on your own infrastructure. If everything is fine you can increase the time and the frequency of CPU profiling.

        One more very important thing: never forget to always stop a started CPU profiling. You can add a timer to automatically call the stop function after a while.

        2) security

        Using the inspector in Node.js it’s like opening the door of the core of your application. You should be very careful about who can use features like CPU profiling and/or the debugger. Never make the inspector “public” as being able to launch a feature from an unsafe route (not protected with an authentification mechanism). Even the collected data can be seen as critical, never send it to a system you do not trust.

        Conclusion

        CPU profiling is really a must-have tool for every developer. And now, with some precautions, we can run it on production thanks to the amazing work done by the V8 and Node.js team.

        The inspector module offers a lot more features than you can use to debug your application.

        I will write another article about using CPU profiling and the inspector on production on a high traffic project.

        Sources & links

        Voodoo Engineering

        Learn about Voodoo’s engineering efforts.

        Thanks to Clint FENTON. 

        Vincent Vallet

        Written by

        Vincent Vallet

        Voodoo Engineering

        Voodoo Engineering

        Learn about Voodoo’s engineering efforts.

        Write the first response

        More From Medium

        More on Nodejs from Voodoo Engineering

        More on Nodejs from Voodoo Engineering

        WebSockets on production with Node.js

        Vincent Vallet
        Mar 11 · 5 min read

        42

        More on Nodejs from Voodoo Engineering

        More on Nodejs from Voodoo Engineering

        A successful Voodoo.io NodeJS meetup on 13th November!

        Aymeric Roffé
        Nov 20, 2019 · 3 min read

        5

        Related reads

        Related reads

        Request Id Tracing in Node.js Applications

        Andrey Pechkurov
        Dec 6, 2018 · 5 min read

        827

        Discover Medium

        Welcome to a place where words matter. On Medium, smart voices and original ideas take center stage - with no ads in sight. Watch

        Make Medium yours

        Follow all the topics you care about, and we’ll deliver the best stories for you to your homepage and inbox. Explore

        Become a member

        Get unlimited access to the best stories on Medium — and support writers while you’re at it. Just $5/month. Upgrade
        AboutHelpLegal
        ================================================ FILE: test/test-pages/lazy-image-2/expected-metadata.json ================================================ { "title": "The Spectacular Story Of Metroid, One Of Gaming's Richest Universes", "byline": "Mama Robotnik", "dir": null, "lang": "en-us", "excerpt": "Nothing beats the passion of a true fan writing about something they love. That's what you're about to see here: one of the richest, most amazing tributes to a great gaming series that we've ever run on Kotaku. Warning #1: this one might make your browser chug, so close your other tabs. Warning #2: This piece might make it hurt a little more than there are no new Metroid games from Nintendo on the horizon.", "siteName": "Kotaku", "publishedTime": "2013-09-11T10:00:00-04:00", "readerable": true } ================================================ FILE: test/test-pages/lazy-image-2/expected.html ================================================

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        Nothing beats the passion of a true fan writing about something they love. That's what you're about to see here: one of the richest, most amazing tributes to a great gaming series that we've ever run on Kotaku. Warning #1: this one might make your browser chug, so close your other tabs. Warning #2: This piece might make it hurt a little more than there are no new Metroid games from Nintendo on the horizon.

        Please note that this is the first half of Mama Robotnik's massive Metroid story. The second half can be found here. The entire post is a greatly-expanded version of a post that Mama Robotnik originally published on the NeoGAF forum before revising and reworking it for Kotaku. Take it away, MR...

        Nintendo’s Metroid series tells us of a malevolent and vicious universe. It’s a maelstrom in which benevolent races are routinely extinguished, and corrupt empires wage war for ownership of living weapons.

        It’s a place in which xenocide is a commissioned service, and grievances are resolved with planetary apocalypses. Everything is chaotically connected to a dead race of avian prophetic poets fighting a war throughout the cosmos. It’s a dark place to visit.

        There are two purposes to this article: to explore the expansive lore of the Metroid universe – with speculation to fill in the gaps – and to exhibit some extraordinary Metroid-inspired art. All artwork is credited to its original source – follow the links to see further works of these spectacular artists.

        Notes on Speculation and Lore

        The games tell us much about this hostile universe, but there are a lot of unresolved story points. In response to these mysteries, the article will provide a healthy amount of speculation. You can consider the piece to be either a makeshift timeline illustrated with fan-artwork, or simply an enthusiastic attempt to reconcile the series continuity into a cohesive whole. The article is informed by the extensive research previously performed by its author. The approach taken regarding speculation is thus: The logical inclusion of probable events that resolve mysteries, while maintaining the themes of the series.

        Before we begin, let’s briefly revisit the five points of essential lore:

        • Metroids are a genetically-engineered species, created by the Chozo in the prehistory of the games. By the time the first entry begins, Metroid creatures only exist on the planet SR388. At some point long before the games, there was also a Metroid presence on the planet Phaaze.

        (Metroid IL Return of Samus, Metroid Prime III: Corruption and Metroid Fusion)

        • At least some Chozo possessed a native ability to see into the future.

        (Metroid: Zero Mission and Metroid Prime)

        • The Chozo discovered the living planet Phaaze with their Elysian Research Outpost. We are not told what transpired immediately after this discovery, but something happened that caused at least one Metroid organism to appear on Phaaze. The planet then loaded this creature – along with pieces Chozo-style powersuit armour, into a Phazon seed and launched it towards a heavily populated Chozo planet. This seed impacted Tallon IV and is contained by the Chozo within an impenetrable shield. The mutated superevolved Metroid creature within – clad in crafted power armour – is trapped in the shield until Samus Aran deactivated it thousands of years later.

        (Metroid Prime EU release, Metroid Prime III: Corruption and Metroid Prime Trilogy)

        • There are living planets in the Metroid Universe. Phaaze is explicitly referred to as being alive, and could interact with a sentient mind – as shown when a Galactic Federation Aurora Unit is implanted. The planet SR388 could be interpreted as having some form of sentience – it shook with apparent anger when its creatures were killed by Samus Aran, and precisely manipulated its oceans to lure the bounty hunter into hostile situations. (Metroid II: Return of Samus and Metroid Prime III: Corruption). The immediate backstory to the first game in the series is the discovery of the planet SR388. The final event of the final game in the chronology is the final destruction of SR388.

        (Metroid, Metroid II: Return of Samus)

        Referencing

        Each story section includes one or more of the below superscript annotations, to help inform the reader as to where the lore or speculation comes from. A brief key:

        With all that said, let us begin.

        Part One: The Wars in Heaven

        The Living Planet

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        On an unknown planet in the universe, a race of avian humanoids evolved. The species that will come to be known as the Chozo possessed great strength, agility and intelligence. The species is peaceful, and is driven by a social/religious value that nature is sacred. [M1 / MP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        Certain blessed individuals were born with a unique gift – the vague comprehension of events set to take place in the distant future. Driven by these prophecies, the race advanced quickly and became space faring. With abstract predictions of a hostile universe, the Chozo developed powered armour and armaments to defend themselves. Prepared for whatever hostility awaited them, the Chozo explored the stars. [M1 / MP / MP SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Elearia)

        The Chozo discovered that – despite their prophets’ visions of a chaotic and warring universe – the cosmos was enjoying a prolonged period of peace and enlightenment. First contact was made with a number of old and wise races, such as the Ylla, the N’kren, the Bryyonians, the Alimbic and the Luminoth. The species shared their cultures and technology, and gently colonised wild worlds such as Aether, Elysia, and Tallon IV. [MP / MPH / MP2 / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Slapshoft)

        Peace reigned through the cosmos. The alliance was a great universal renaissance, and lasted for a millennium. [MPH SP / MP2 SP / MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        During this calm, the Chozo prophets continued to receive increasingly severe visions of chaos. They foresaw a universe consumed by war, horrors evolving on distant worlds, and a great toxicity waiting to be unleashed. As the visions became more precise, the species isolated itself from its allies. The Chozo civilisation became intensely driven to fight this unclear threat. [MP / MP3 SP / M2 SP /MF SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: DanilLovesFood)

        The Chozo needed more potent tools to locate this unseen and distant danger. They expanded their SkyTown colony on the gas giant Elysia and remade it into a vast interstellar observatory powered by the planet’s endless storms. The facility was of such scale that an entire species of artificial life became necessary to maintain it. The Chozo created their first species – the mechanical Elysians. [MP3 / MP3 SP]

        Probes were launched across the universe, and the Elysians and Chozo scrutinised the data. The search took generations, while the planet’s tempestuous atmosphere battered SkyTown, weathering the station faster than the Elysians could maintain it. After countless probe launches, a partial transmission received from a decaying and distant satellite set prophecy in motion. [MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Mechanical-Hand)

        The data received was terrifying. The blue planet registered as an organism, somehow existing as both mineral and flesh. Impossible radiation pulsed from the surface, which overwhelmed the Chozo satellite and rendered it inert. The location of the planet was immediately lost, and only a broad region of space could be established. [MP3]

        With this find, the Chozo purpose on SkyTown was fulfilled. The race departed the facility, leaving the Elysians to continue their monitoring of the stars. The abandoned race of robots continued to launch satellites to try and rediscover the blue world, hopeful that such a discovery would herald the return of their Chozo creators. The Elysians searched unsuccessfully until Elysia’s endless storms eroded their civilisation into a rusted remnant. [MP3]

        The Chozo reconciled their vague discovery of a blue living planet with their prophecies of toxicity. On this distant world of poison, could creatures have evolved so vicious that they endangered the universe? [MP3 SP]

        The Invasion of Phaaze

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        Finding the exact location of the deadly planet becomes a priority for the Chozo civilisation. A gargantuan ship was assembled on the holy planet of Tallon IV, and dispatched to the dark corner of the universe where the Elysian satellite had been lost. The greatest Chozo warriors, scientists and prophets commenced a crusade for the hostile world, knowing that they would likely never make it back home. During their long journey, they conceive a name for their target: Phaaze. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: SesakaTH)

        Generations passed, and the Chozo expedition finally located the blue planet. As they approached, they witnessed the living world as it endlessly pulsed with blue and white energies. There was nothing like this place elsewhere in the universe. [MP3 SP]

        Their scans confirmed their worst fears – this atmosphere was a bath of radiation and mutation – and evolution had produced horrors. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: SamusMMX)

        For billions of years, Phaaze had mutated and irradiated life that evolved on its surface. The strongest creatures had survived to thrive in an ecosystem of beautiful poison. It was then that the Chozo understood: They had arrived at the home of the most devastating and deranged creatures in the known universe. [MP3 SP]

        If these monsters were to escape their containment on Phaaze, they would voraciously consume their way through the cosmos. With younger races only centuries away from space travel, the Chozo could not risk them finding this world and releasing its terrors. [MP3 SP]

        The Chozo expedition came to an impasse. The threat of Phaaze’s superpredators had to be neutralised, but severe action against the planet would be sacrilege. The Chozo held life sacred, and refused to destroy the unique living world. [MP3 SP]

        A dangerous plan was agreed upon. The expedition ship landed on Phaaze, exposing the crew to tremendous radiation. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Adoublea)

        Chozo Warriors in power suits fought the planet’s creatures as they swarmed the ship. The soldiers battled, watching their kin die around them, in a desperate mission to buy time. [MP3 SP]

        The scientists within the ship began to harness the intense radiation around them, to try and engineer an artificial predator that could neutralise the planet’s superpredators. With access to the unique Phazon mutagen that covered the poisonous world, genetic engineering that should have taken decades was done in days. The Chozo engineered the first Metroid. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Methuselah3000)

        The Metroid creature was unleashed onto the planet, and the radiation caused it to reproduce quickly. The resulting swarm of Metroids began to consume the planet’s monstrosities and established themselves as Phaaze’s apex predator. [MP3 SP]

        The Chozo mission was complete. The worst creatures were being hunted to extinction, and the Metroids were expected to die from starvation soon after. The cost had been enormous – most of the crew had been killed defending the ship, and the survivors were deathly ill from radiation poisoning. The burnt and damaged ship took off for the long journey home, but the crew soon succumbed to the radiation they had endured. The autopilot took the ship of Chozo bodies home. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        On Phaaze, the Metroid presence lasted decades as they consumed the planet’s superpredators. The corpses of Chozo warriors were absorbed into the planet, and their battle armour slowly became weathered and scattered. The planet’s slow sentience developed an outrage that seethed under its continents. It had been violated by the Chozo. As the Metroid infestation began to die out, Phaaze developed a very primitive concept of purpose and retribution. [MP3 SP]

        Phaaze established a vague awareness of concepts it had absorbed from the brains of the Chozo warrior corpses – and the location of two worlds from the Chozo’s memories. As the planet entered its reproductive cycle, it purposely directed two of its seeds towards the planets Tallon IV and Aether. In the seed sent to the Chozo world, Phaaze included one of the last surviving Metroid creatures and some ruined pieces of Chozo armour, intended as a reminder of the crime Phaaze had endured at their hands. The planet sent its second seed to Aether, as the absorbed memories informed the living planet that its inhabitants were friends of the Chozo, and therefore the enemies of Phaaze. [MP 1 / MP 2 SP / MP3 SP]

        The expedition ship – heavily damaged by radiation and lack of maintenance – was guided back to civilisation by an increasingly erratic auto-pilot. After decades it eventually approached the Chozo world of Zebes, and crash-landed onto its surface. The Chozo civilisation attempted to recover data logs from the wreckage with very limited success – they were able to understand the sacrifice that the heroic crew had made, and confirmed the apparent success of the Metroids in neutralising the creatures on the living planet. The Chozo authorities were unable to establish the location of Phaaze, or recover much in the way of scientific data concerning it. [MP3 SP / SM SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Methuselah3000)

        As the Tallon IV seed began its centuries of travelling through space, the lone Metroid within absorbed vast amounts of Phazon and radiation. It became self-aware, and grew in size, intelligence and strength. It used the ruined pieces of Chozo armour to construct itself an exoskeleton, and descended into madness. The exoskeleton failed to protect the creature from the endless radiation, and the Metroid became as exotic as Phaaze’s extinct superpredators: An undying tortured genius. [MP / MP2 / MP3 / MP3 SP]

        The creature that would come to be known as Metroid Prime resented Phaaze for imprisoning it in the Leviathan. It resented the Chozo for creating and discarding the Metroids. It decided that it would survive, bring order to the chaotic universe that birthed it, and somehow enslave Phaaze to its will. In its solitude, immortal as a consequence of its mutations, Metroid Prime plotted its revenge against the universe. [MP / MP2 / MP3 / MP3 SP]

        The Dark Planet

        With a clear understanding of the danger of living planets, the Chozo authority commenced a search for similar threats. With far more advanced technology than their ancestors had during the Elysian era, the Chozo were unfortunate enough to find a planet of even greater horrors. [MP 3 SP / M2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: PeaceFistArtist)

        The Chozo detected strange readings coming from a world in a desolate part of the galaxy. The planet had been previously considered so obscure and unimportant that it didn’t have a name, merely catalogued with the codename SR388 and left to its obscurity. A detailed analysis picked up some extremely strange observations; though seemingly mineral, the caverns and liquids beneath the surface shifted with metabolic rhythm – as if the whole planet was somehow a living thing. A ship was dispatched, and the strongest Chozo warriors braved the caverns beneath the surface. [M2 / M2 SP]

        Few made it back. They told of a cauldron of evil, an environment so hostile and vicious that it had birthed the most terrible things. [M2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: LightningArts)

        Beneath that planet, evolution had been won by an abomination that could steal the flesh, abilities, memories and strengths of all of its prey. The creature was a fusion of energy and plasma that parasitized on life itself. With no word suitable for the nightmare they had discovered, the Chozo simply called it X. If these X-Parasites somehow gained access to the wider universe, there would be no force that could contain them. [M2]

        The threat had to be dealt with. Remembering the apparent success of the Chozo expedition to Phaaze, a plan was put into action. The Chozo assembled their best and brightest, their strongest and wisest. They carved their way into SR388, and dispatched mechanical creatures to construct secure facilities. Robots and Chozo warriors repressed all instances of the X-Parasite as they found them, but casualties were high. The planet appeared to fight the Chozo at every turn, it drowned the invaders in acid and unleashed ambushes of creatures. The endless swam of X-Parasites gained strength from the corpses around them. [MP3 SP / M2]

        Deep in the planet, a glass laboratory was created, its walls highly resistant to SR388’s acid belly. Here, in dangerous proximity to the X-Parasites, the Chozo scientists began their work. [M2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        The Chozo tried to recreate the plan of their ancestors – the use of Metroids to pacify superpredators too dangerous to exist. Without access to the same planetary radiation and materials the Phaaze expedition had, progress was slow. As the war against the planet was raging around them, the Chozo scientists were able to engineer Metroids, but not a variant strong enough to overcome the X-Parasites. As more and more Chozo died protecting the laboratory, a different approach was needed. [M2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Starshadow76)

        The Chozo succeeded in engineering a Metroid Queen, a colossal creature who would lay Metroid Hatchling eggs. When hatched, these resulting Metroids were strong and durable creatures, and finally potent enough to combat the X menace. The Chozo knew that to completely suppress the parasites, the Metroid presence on SR388 had to be permanent. To ensure that the species would not overfeed on the environment and wipe out its food chains, the scientists hardwired an instinct into the Metroid Queen’s feral mind: Only thirty-nine Metroids were to exist on the planet at any one time. This, it was hoped, would keep their numbers high enough to destroy any X re-emergence, but low enough so that they wouldn’t consume the rest of the life on the planet, and starve to death from lack of food. [M2]

        The scientists assembled a payload of Hatchling Eggs and the surviving Chozo warriors distributed them across the planet. The eggs hatched quickly, and the X-Parasites were immediately overwhelmed by the infant Metroids. The X-Parasites were quickly hunted to near-extinction, with only a few surviving cells entering a state of suspension deep in the planet. [M2 SP / MF]

        The Chozo had won their war, but only just. Most of the warriors and scientists had not survived, and those that were left had to make sure that the X-Parasites had been permanently suppressed. The planet shook with tremors; the earth shifted and acid poured, as if the world was trying to crush the Chozo in their glass laboratory. [M2 SP]

        The X-Parasites did not return, and the Metroid Queen continued to scream as her glass prison shook. The Chozo didn’t realise it, but her despair was being heard. [M2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Hermax669)

        SR388 had been violated by the Chozo. Though very different to Phaaze, SR388 had its own vague sense of awareness. It perceived the Chozo as a viral infection, and the dead X-Parasites as part of itself. It understood loss, and shook with ancient rage. [MP3 SP / M2 SP / MF SP]

        It changed itself to change the Metroids. It adopted them to replace the X-Parasites, and quickly killed the weaker breeds. It moved its radioactive minerals closer to their eggs and soon mutated the species. As SR388 had done with X, it did with the Metroids. It made them strong. [M2 SP / MF SP]

        Alpha, Gamma, Zeta and Omega Metroids spawned quickly, and responded to the screams of their Queen. With their bulk and strength, they smashed through the glass laboratory and slaughtered their Chozo creators. The Chozo warriors were hunted down and crushed. [M2]

        SR388 developed into a new cauldron of hostility. The Metroids served as the apex predator, and the robots of the Chozo decayed into machine madness and prowled the ruins, killing on sight. The Chozo mission to suppress the X-Parasite had been a success, but the planet had gained its revenge. [M2 / M2 SP / MF]

        Part Two: The End of the Renaissance

        The Holy World

        The Chozo had devastated two planets for the good of the universe, and sustained many causalities. The superpredators of Phaaze were extinct and the X-Parasites were permanently suppressed. With the crisis over, the race became consumed with a collective sense of guilt over their necessary actions. The Chozo believed the life of the universe to be sacred, and had to reconcile their aggressive actions with their faith. [MP SP / MP3 SP / M2 / MF]

        Worse still, their prophets continued to have visions of endless conflict and death. War was coming to the universe, and it seemed that their sins had not saved them. Many began to doubt these visions, and a schism occurred. [MP/ MP3 SP]

        The bulk of the Chozo civilisation retired themselves from galactic affairs, leaving only a few scattered colonies amongst the stars. The race retreated to the holy planet of Tallon IV, to shun their technologies and begin simpler, poetic lives. These Chozo reconnected themselves to the natural world and tried to find a harmony with it. As time went on, the most potent prophets became manic, and tried to warn their fellows of a great poison that was to come. [M1 / MP]

        These visions were met with increasing dismissal, but the day finally came when the prophets were believed. After eons swimming in the stars, Phaaze’s seed entered the Tallon system. [MP / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Hameed)

        The Leviathan crashed down, and rained poison and death unto the world. The impact survivors watched as their sacred nature succumbed to the mutagens leaking from the seed, and barricaded themselves in their temples as the flora and fauna transformed. Phazon spread beneath the surface of the dying planet, and radiation storms battered the surface. [MP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Source: Riivka)

        The Chozos’ punishment for their sins, and the fulfilment of Phaaze’s wrath, reached biblical proportions. The Chozo of Tallon IV did not get to rest in peace. Their life energies suffered from Phazon disruption, and upon death they became mad ghosts who screamed forever as they were torn in and out of the material world. In this purgatory, the undead immaterial Chozo murdered anyone they could find. [MP / MP3 SP]

        As their numbers dwindled, the last of the Chozo constructed a great temple above the impact crater. Within this temple, they used what little technology remained to project an energy field around the Leviathan to slow the spread of contagion. As the Chozo civilisation on Tallon IV was extinguished, their dying prophets told of a hero who would one day emerge, to enter the crater and defeat the evil worm within. [MP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Source: Havoc-DM)

        Within the Impact Crater, Metroid Prime remained trapped within the Chozo energy field. In its armour constructed from ancient Chozo power suits, it continued its wait to be unleashed on the universe. [MP / MP3 SP]

        Dark Echoes

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        On the planet Aether, an ancient race of mystics known as the Luminoth received the horrifying data coming from Tallon IV. In distant times, the Luminoth and the Chozo had been steadfast allies – until the Chozo retreat ended their ties. Desperate to assist, the Luminoth began to organise a rescue mission. [MP2 / MP2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: PugOfDoon)

        A dark transmission was received from Tallon IV. The image showed a screaming, ghostly Chozo figure, flickering in and out of the living universe. In its undead madness, it spoke for its kin. It raged that they would kill anyone who would set foot on their world. The planet was pandemonium, a cursed world on which the dead could not die. As the signal faded, the Luminoth realised that there was no one left alive to rescue. [MP SP / MP2 SP]

        The Luminoth were receiving strange readings from the devastated planet. A mutagen was spreading, unlike anything they had ever encountered. They scanned the stars for its source, and made a devastating discovery – a mass of the same mutagen was on a collision course with Aether. Phaaze’s second seed had nearly arrived at its destination. [MP2 SP / MP3 SP]

        The people of Aether turned to their technology to save them. Their planet had no native star of its own, and had been implanted millennia ago with a complex energy network that sustained all life. This system was reverently called the Light of Aether, and harnessed the light of the universe in its mechanism. The Luminoth realised that even with this great power, they could not destroy the Phazon Leviathan. A different approach was needed. [MP2 / MP2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: SesaKath)

        The Luminoth used their great Light to engineer a small pocket universe, a dark lifeless echo of existence. The plan was bold: they would use the Light of Aether to surgically open the fabric of reality in the path of the Phazon seed, and allow it to harmlessly enter the pocket universe. If all went well, they would be saved. [MP2 SP]

        The day came, and the Leviathan entered Aether’s atmosphere. The Luminoth commenced their great plan. [MP2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Adriencgd)

        Phaaze’s seed was a sum of living materials beyond Luminoth comprehension. It hit the pocket universe with incalculable force, and a tsunami of exotic energy ruptured space and time. The equipment containing the dark reality lost containment within moments, and the Luminoth were helpless as their creation expanded across the entire planet. A wave of dark energy absorbed creatures, structures and land into the dark universe, and what was once a single planet – was now two. [MP2 / MP2 SP]

        The Luminoth surveyed the devastation. The Phazon seed was gone – it had indeed collided with the dark universe. Entire continents, with millions of inhabitants, had vanished with it. [MP2 / MP2 SP

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: AzureParagon)

        In the dark universe, a grotesque world was being born. Previous inhabitants of Aether, having been absorbed when containment of the pocket universe was lost, found themselves twisted by the corrosive new reality around them. Most perished, and their flesh fed the strange carnivorous fungi that glowed sickly colours. Some survivors were mutated by the Phazon slowly spreading beneath the surface, and adapted to survive in the hostility. [MP2 SP]

        Aether and its echo, the Phazon-infested Dark Aether, existed in synchronicity. As the Luminoth tried to rebuild their planet, it took only decades for cracks to form in the ether separating the two realities. As rips in the universe shattered open, Aether became a battlefield. [MP2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Xxkiragaxx)

        A womb of Phazon mutation and dark energies had birthed a cunning and ferocious horde. The Ing erupted through the cracks between the two worlds, and commenced slaughter. They were fought back by the Luminoth, and a war began between the two parallel worlds. The Ing invaded Aether with regularity, and killed, pillaged and destroyed all that they could find. The Luminoth retaliated and crusaded into Dark Aether in their Light Suits, on suicide missions to exterminate the source of the Ing menace. Both sides suffered colossal casualties as the decades went on. [MP2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        The war was being lost by the Luminoth. The Ing had exterminated most of their race and had stolen too many vital technologies. With the theft of essential energy components from the Light of Aether power network, they had become a defeated people. [MP2]

        The Ing had destroyed all of Aether’s ancient ships, and condemned the Luminoth to no escape from their doomed world. With no other choice, the survivors sealed themselves in an inner sanctum, and entered a state of suspended animation. One custodian, U-Mos, volunteered to be their guardian. As Aether became weaker and weaker, the Luminoth waited for someone to save them. They would wait a very long time. [MP2]

        The Sacrifice of the Alimbics

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Kihunter)

        As the Chozo and the Luminoth fell, so too did other ancient races. In a distant part of the universe, the Alimbics were a militaristic society that maintained peace in their galactic cluster. Their order was shattered when a murderous entity, originating from someplace beyond the understood universe, plummeted into one of their worlds. The creature emerged from the devastation as a gaseous entity, and assumed an Alimbic-styled body to begin its onslaught. [MPH]

        This alien juggernaut was named Gorea by the Alimbic race, and they soon understood it brought only death. Gorea killed every Alimbic it could find, and destroyed everything in its path. Planet after planet fell to Gorea, and the Alimbics realised the creature would never stop. [MPH]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Sesakath)

        The Alimbics performed an act of supreme sacrifice. They combined the mental energies of their entire race to forge a prison for Gorea. The psychic prison held it bound, and it was transplanted into an organic vessel called The Oubliette. The vessel was launched into the void outside of the universe, a course that would keep its indestructible prisoner in exile forever. The systems of the prison ship were tasked to scan the every molecule of the imprisoned Gorea, and devise an Omega weapon that could be used to kill it. [MPH / MPH SP]

        The mental energies expelled in this plan consumed the physical bodies of the entire Alimbic race. They vanished from the universe in an instant. Their sacrifice protected all life in the cosmos from Gorea’s murderous rampage. [MPH]

        The War of Bryyo

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Sesakath)

        As the old races of the universe died around them, the lizard people of Bryyo faced their own challenges. The Bryyonians were an advanced, space-faring race who had learned much from their Chozo allies. Their society was a deeply polarised one, with tensions eternal between the scientific and religious factions.[MP3]

        Over the previous centuries, the scientific agenda had dominated, with space travel proving beneficial and enlightening. As the Chozo, Luminoth and Alimbics faced extinction, the religious Bryyonians believed more than ever that the universe was a hostile place, and became desperate to stop their scientific counterparts. [MP3]

        A great war exploded across Bryyo. By its end, the scholars had been wiped out and the survivors of both sides had regressed to a feral existence. The race devolved into animals, wandering around ruins that they no longer understood. Language vanished and strength ruled. Anyone who landed on Bryyo was meat, to be killed and eaten. [MP3]

        The Little Rainy Planet

        The onslaught of vengeances, conquerors, poisons and politics destroyed the old races. The Alimbics had lost their flesh, while the Bryyonians had lost their souls. The Luminoth had retreated into stasis, and the Chozo of Tallon IV had been condemned to a living death. [MP / MPH / MP2 / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Kaiquesilva)

        On a small, rainy planet called Zebes, the last known Chozo colony had watched the stars with impotence. This small settlement of the nearly-extinct avian race witnessed the end of the great universal renaissance, and the slow beginning of a new chapter in galactic history. Gradually, the younger races were launching their first satellites into space. In time, new empires would rise to take the place of the old. [M1 / M1 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        Zebes prophets’ saw the visions the Chozo had always endured: great wars, spreading poison and death. And suddenly, something bold was foreseen. [M1 SP / MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Fddt)

        A great hunter, clad in orange, red and green. The Chozo glimpsed a future hero, alone in the darkness beneath worlds, fighting so that good could survive evil. They saw her curing poisoned planets, and ending galactic wars. They saw the universe’s one chance to survive its apocalyptic future. They saw the only one who could defy prophecy. [M1 / MP3 SP]

        And they saw her wearing Chozo armour. [M1]

        Part Three: The New Empires

        The Humans

        On the planet Earth, the human race had finally developed a ship capable of leaving their solar system. A brave crew ventured into the universe to learn whether life existed elsewhere. Their discoveries fundamentally changed the human condition. On planet after planet, they found ruined tombs and cities, guarded by weathered statues of dead races. Most significant of all, they found technology. [M1 SP]

        The humans reverse engineered their salvage, and advanced with pace. Within another century, faster-than-light ships explored the stars, and colonies transformed hostile worlds into homes. Peaceful relations formed between other younger races, and a great Galactic Federation was founded. [M1 SP]

        The Space Pirates

        In a less hospitable region of space, a cabal of battered races joined their forces to survive. On planets where acid rain burned flesh and magma flowed, the alliance expanded into a hardened space empire. They ventured into nearby systems and took what they needed from anyone they could reach. They found the ruins of the old races and ransacked the ancient technologies within. They immersed themselves in science and unlocked the secrets of their finds. Within decades, they had advanced their spread with stronger and faster ships. The creatures enhanced themselves, rewriting their genetics and integrating mechanisms beneath their flesh. They were unique: a cybernetic race of furious murderers with a skill for patient scientific process. As more planets were invaded, their conquered civilisations were conscripted by force. [M1 SP / MP / MP3]

        The inevitable moment came when their Empire reached the borders of the vast Galactic Federation. [M1 SP / MP / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Mr-Corr)

        First contact was brief and furious. On that day, the warning went out to all the worlds of the Federation: Beware the Space Pirates. Though no state of war was officially declared, the empires attacked each other on sight. The Galactic Federation was large enough to repress any meaningful incursions into their space. [M1 SP / MP SP / MP3 SP / SM SP]

        The Massacre of Two Families

        The Galactic Federation discovered the last Chozo Colony on Zebes. The tired, ancient avians welcomed the humans and shared with them wisdom and knowledge. They offered the Galactic Federation new sciences, and taught them how to make organic computers. The Federation studied the Chozo’s own central processing unit, an engineered brain that mothered over their colony, and left with plans to assemble their own variants. On the nearest habitable planet of K-2L, a colony was established. [M1 / MP3 SP]

        On this world, the human Samus Aran was born. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Methuselah3000)

        Barely out of infancy, the young Samus witnessed her family die. A Space Pirate raiding party overwhelmed her colony and murdered everyone she ever knew. By staying silent while surrounded by horror, Samus survived as the Pirates ransacked the settlement and left. [M1]

        The Chozo colony on Zebes received K-2L’s automated distress signal. In an ancient dusty ship, they reached the planet and found Samus to be the only survivor of the massacre. The child was brought to Zebes, and the Chozo deliberated. Should she be returned to her own kind, or allowed to stay? [M1]

        Across the colony, the Prophets experienced a simultaneous moment of clarity. They understood immediately that they had found their prophesised hero. The young girl was their inheritor, and would grow strong. She would learn all she could from them, and take their strongest technologies into the universe. She would be the hero against the oncoming storm. [M1 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: R3dFiVe)

        Samus Aran reached maturity amongst the Chozo. She was trained in the combat arts of the great extinct races. She was infused with Chozo genetic material so she could employ their technologies. She was educated to be a scientist, an explorer, and a tactician. Everything that was good about the Chozo civilisation was allowed to live on in Samus. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Pyra)

        Samus became an adult, and the Chozo presented her with their greatest works: a toughened power suit and an agile spacecraft, both more potent than anything their race had ever made. The Chozo leader, decaying and blind, told Samus it was time for her to find her destiny in the universe. Samus Aran departed for the stars, and years pass. [M1 / M1 SP]

        As Samus tried to reconnect with her heritage on Earth, the last Chozo prophets on Zebes received a final vision: The Space Pirates were coming for them. It was time for the last Chozo to be extinguished from the universe. [M1 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Phobos-Romulus)

        The Chozo hid their technologies throughout the planet, in places that they were certain Samus would find them. They concealed a second Power Suit within the walls of their holy temple, having foreseen that Samus may require it in the future. They then returned to the surface to await the inevitable. [M1 SP]

        The Space Pirates invaded in force, and murdered Samus Aran’s second family. The Chozo became extinct. [M1 / MP SP]

        The Mother Brain

        Space Pirate scientists arrived shortly after the carnage and focused their attention on the legendary Chozo organic central processing unit. They rewrote its benign programming and injected stimulants into its flesh. They enabled it to form an artificial intelligence obsessed with strategy and conquest. They drove its computational potential towards absolute advancement of the Space Pirate Empire. [M1 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Jaagup)

        The results went beyond High Command’s most optimistic projections. The Space Pirates had created a leader, a desperately needed figure to unite their fragmented empire. They had created their Mother Brain. The great Space Pirate generals Ridley and Kraid arrived at Zebes, ready to pay tribute to their new master and to plan for the future. Mother Brain delivered to the Space Pirates knowledge and power. She told them of a world referenced in her oldest Chozo databanks, a planet bathed in a mutagenic poison waiting to be farmed. She instructed High Command to prepare an armada of ships and invade the planet Tallon IV. [M1 / MP SP]

        The order was followed immediately, and the High Command discovered a world deranged by contagion. Beneath its surface, endless pools of Phazon waited to be weaponised, and a great mining operation began. Mother Brain received data from their readings on the planet; even after thousands of years, the source of the Phazon was still contained in the Chozo force field. She scrutinised her records further, and was unable to ascertain any method of breaching the barrier. The Space Pirates could retrieve the Phazon, but were denied access to its source. [MP]

        The Metroids

        A perfect storm brewed. As the Space Pirates gained access to the most potent mutagen in the universe, the Galactic Federation made an equally eventful discovery: They found the dark planet SR388. [M1 / M2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Fireborn Form)

        A Galactic Federation survey team studied the surface, and soon encountered a gelatinous creature that swam through air. The alien defied gravity and physics as it phased through dense rock with ease. It perceived the survey team, and made a few curious chirps in their direction. It then suddenly changed temperament, aggressively charging to latch itself onto the skull of one of the party. The victim died in agony as the Metroid fed on all the energy within, and could not be removed until its prey had been reduced to a dried husk of collapsing matter. The young Metroid had just killed, in a way that science could not explain. [M1 SP / M2 SP]

        With effort and casualties, the scientists contained a few infant specimens of the Metroid creatures, and left the planet without further incident. [M1]

        As their vessel went back to the stars, SR388 was aware their withdrawal. It harboured a great contempt for the invaders, an endless hate fuelled by the impotence it endured centuries ago when the Chozo had committed their great invasion. The living planet had spent centuries honing the Metroids into perfect killers, and knew the devastation they could cause upon maturity. The planet had intentionally allowed the humans to take a few Metroids away so that the creatures could grow up and kill anyone out amongst the stars who ever thought of returning. SR388 took any opportunity to gain revenge against an outside universe that refused to leave it alone. [M2 SP]

        As the scientists began to broadcast their findings back to the Galactic Federation, Mother Brain intercepted the transmission. She cross-referenced their data with notes buried in the Chozo’s ancient fragmented records. She deduced that the Metroids were a form of genetically engineered predator of incredible power, created by the Chozo for an unknown purpose. Mother Brain ordered High Command to get the creatures to her by any means necessary. [M1 SP]

        The Space Pirates overran the Galactic Federation vessel and stole the Metroid creatures. They divided their prize: some were sent to their nearest Homeworld; others were sent to the Tallon IV outpost; and the most potent were delivered straight to Zebes for the experiments of Mother Brain. [M1 / MP / MP3]

        With the arrival of the first Phazon samples from Tallon IV, the exotic substance allowed the Space Pirates to slowly produce stable cloned Metroids across their breeding sites. [M1 SP / MP SP]

        The Revenge of Samus Aran

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Ojanpohja)

        In her first mission as a Bounty Hunter, Samus Arran was commissioned by the Galactic Federation to neutralise the stolen Metroids. Through careful investigation, Samus discovered that the Pirates are operating from Zebes – her home. She concluded that the Space Pirates had murdered her second family, as they had done with her first. They have took from her everyone she ever loved, and destroyed her two worlds. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Stuart Hughe)

        Samus stormed Zebes and killed everyone in her path. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Immarart)

        As her defences were breached, Mother Brain unleashed the great generals Ridley and Kraid. Both were killled, and, desperate to stop the intruder, Mother Brain released the Metroids. Samus Aran exterminated the creatures, and invaded the inner sanctum. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Twigs)

        Samus confronted the malevolent Mother Brain and blasted apart her body. A power overload was caused, and the Tourian facility shook itself apart. Samus evacuated to her ship and tried to leave Zebes, but a Space Pirate battleship in orbit registered her ascent and opened fire. Samus’ gunship plummeted back towards the Zebes and impacted Chozodia, her former home. [M1]

        Extremely lucky to be alive, Samus crawled out of the remains of her destroyed power suit, and fled as Space Pirate forces stormed the area. Samus hid, crawled and ran to find sanctuary in the deepest part of the Chozo’s most revered temple. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Eyes5)

        Samus found herself surrounded with murals of the dead Chozo, and accepted she was alone in the universe. Overcoming despair, she solved the trials of the Chozodian temple and a concealed power suit was revealed to her. This shining armour was even more potent than the one she had just lost, and was able to integrate the most exotic Chozo technologies. Samus realised the greater meaning of her find; the Chozo had left her gifts for her in places they had foreseen she would traverse. Her adopted family continued to protect her long after their deaths, and she would find their statues cradling survival equipment in the darkest corners of the cosmos. [M1 / MP / MP3 / M2 / SM]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Imachinivid)

        With her new armaments, Samus cleansed the Space Pirate presence from Zebes. She came to be known as “The Hunter”, and the Space Pirates learned that they will always be hunted down for what they did to her families. They fled the planet in terror. [M1 / MP / MP2 / MP3]

        Tallon IV

        As years passed, Samus Aran accepted further missions from the Galactic Federation. The bounty earned funded her personal vendetta against the Space Pirates. She improved her armaments, paid for black market information and stormed their outposts. Samus showed her enemies no mercy, and became the feared nemesis of their entire civilisation. With the income from her Federation services, Samus had soon amassed enough money to buy the most secret information regarding the Space Pirates: the coordinates of their stronghold on an old forgotten planet called Tallon IV. [MP1 SP]

        Samus guided her ship into the Tallon system and investigated an orbiting space station. She discovered a failed genetic engineering facility whose Space Pirate crew was murdered when they lost control of their own creations. Samus fought her way through the ferocious beasts scattered within, and discovered a half-insane cyborg recreation of the Space Pirate general Ridley. As the station began to collapse, the biomechanical dragon fled to the world below, and Samus pursued. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Lightningarts)

        Samus lost Ridley in the planet’s stormy atmosphere, and elected to land in a nearby jungle to conceal her presence from the Pirate ground forces. Exploring the surroundings, Samus discovered that the planet was once home to the bulk of the extinct Chozo civilisation. In a great temple Samus studied poetic murals that told of the Phazon comet that had struck their world. The scribblings informed her of a creature trapped deep in the comet that they referred to as “The Worm,” and of the powerful shield they erected to prevent its escape. Samus read their last prophecy; that a hero would traverse fire and ice, jungle and cave, and find twelve sacred keys that would deactivate the barrier and allow passage to the Impact Crater. This saviour from the stars would bring down the ancient shield, and destroy the worm that infected their planet. [MP1]

        She continued her exploration, and battled ferocious flora and fauna. The Hunter came to understand that the Space Pirates had established a complex military installation that descended far below the surface. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: R-Sraven)

        Samus hunted the Pirates and accessed their computer logs. The Empire had found quantities of an intensely potent mutagen called Phazon. Laboratories across the outpost experimented with the substance, and in a short space of time they had created prototypes for the next generation of their races: powerful Phazon-fuelled juggernauts. Should these advances continue, Samus knew that the Space Pirates would be able to conquer the Galactic Federation. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Greenstranger)

        In the most secure laboratory, Samus made a devastating discovery. The Space Pirates had used Phazon to create an army of stable clone Metroids and lost containment. The Metroid creatures were roaming the caverns deep in the planet, reproducing and mutating as the Phazon influenced their physiology. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Ohimseeinstars)

        Samus’ final discovery was the most horrific. The powerful, poisonous Phazon was not a rare material on Tallon IV. Despite the Chozo shield containing the Impact Crater, the substance had spread and consumed the world inside-out. The core of the planet presented the Space Pirates with a vast supply of Phazon, enough to fuel their conquest of the stars. [MP1]

        Samus destroyed the mining facilities and laboratories, and reconstructed the twelve parts of the ancient Chozo cipher. She destroyed living weapons such as the Thardus experiment, and annihilated the prototype Omega Pirate. She overcame corrupted Metroids, and banished the tormented Chozo ghosts from the living world. She fought the mad Meta Ridley, and on his demise deactivated the Chozo containment shield. As prophesised, Samus Aran entered the Impact Crater. [MP1]

        The Worm

        Samus Aran had opened Metroid Prime’s cage, and had no understanding of what she was about to unleash on the universe. The creature had been imprisoned in a different era, and had spent eons being tortuously transformed by Phazon into an undying mad genius. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Chrysaetos-Pteron)

        Samus and the ancient Metroid battled, and the bounty hunter shattered the creature’s metal armour. By channelling the surrounding Phazon deposits into a supercharged energy beam, Samus was able to devastate Metroid Prime’s gelatinous body. After a tremendous battle, the old creature began to collapse on itself. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Sabretoontigers)

        Seemingly dying, Metroid Prime lashed out, grabbing a layer of material from Samus Aran’s armour. The creature melted into a pool of Phazon particles, and the bounty hunter evacuated the Impact Crater. [MP1]

        Samus Aran had seemingly succeeded in her mission. The surviving Space Pirates abandoned their devastated facilities and hastily evacuated the planet. With the defeat of Metroid Prime, the Phazon contagion was slowly stopping its spread. The tormented Chozo spirits that had been bound to the planet were finally able to achieve their rest. Leaving the world to recover from its devastation, Samus Aran headed back to the stars. [MP1 / MP1 SP]

        The Dark Hunter

        Metroid Prime’s exposure to millennia of Phazon had given the creature extremely exotic abilities, the most potent being its durability – to recreate itself after nearly any level of destruction. As it had collapsed on itself, the essence of Metroid Prime craved the strength and adaptability present in Samus Aran. In the Talon IV impact crater, Metroid Prime recreated itself as a dark copy of the woman who had defeated it. [MP 1 / MP2 / MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Imachinivid)

        Dark Samus clawed its way out of the Impact Crater. It departed Tallon IV to spread its venom across the stars, and to sow the seeds of a great war. [MP1 / MP2 / MP3]

        Click here for the second half of this epic story.

        Mama Robotnik is a video game historian living somewhere in the British Empire. He specialises in unearthing lost gaming media, but also enjoys a good long essay about his favourite games every now and then. He drinks a lot of tea, and has a horrendously naughty black and white cat called Blossom. If you would like to contact him, he responds to his private messages over at NeoGAF.

        ================================================ FILE: test/test-pages/lazy-image-2/source.html ================================================ The Spectacular Story Of Metroid, One Of Gaming's Richest Universes
        Gaming Reviews, News, Tips and More.
        Shop Subscribe
        Gaming Reviews, News, Tips and More.

        The Spectacular Story Of Metroid, One Of Gaming's Richest Universes

        Mama Robotnik
        •
        Filed to:metroid
        metroidkotaku longreadsnintendo
        158
        52
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        Nothing beats the passion of a true fan writing about something they love. That's what you're about to see here: one of the richest, most amazing tributes to a great gaming series that we've ever run on Kotaku. Warning #1: this one might make your browser chug, so close your other tabs. Warning #2: This piece might make it hurt a little more than there are no new Metroid games from Nintendo on the horizon.

        Please note that this is the first half of Mama Robotnik's massive Metroid story. The second half can be found here. The entire post is a greatly-expanded version of a post that Mama Robotnik originally published on the NeoGAF forum before revising and reworking it for Kotaku. Take it away, MR...

        Advertisement

        Nintendo’s Metroid series tells us of a malevolent and vicious universe. It’s a maelstrom in which benevolent races are routinely extinguished, and corrupt empires wage war for ownership of living weapons.

        It’s a place in which xenocide is a commissioned service, and grievances are resolved with planetary apocalypses. Everything is chaotically connected to a dead race of avian prophetic poets fighting a war throughout the cosmos. It’s a dark place to visit.

        There are two purposes to this article: to explore the expansive lore of the Metroid universe – with speculation to fill in the gaps – and to exhibit some extraordinary Metroid-inspired art. All artwork is credited to its original source – follow the links to see further works of these spectacular artists.

        Advertisement

        Notes on Speculation and Lore

        The games tell us much about this hostile universe, but there are a lot of unresolved story points. In response to these mysteries, the article will provide a healthy amount of speculation. You can consider the piece to be either a makeshift timeline illustrated with fan-artwork, or simply an enthusiastic attempt to reconcile the series continuity into a cohesive whole. The article is informed by the extensive research previously performed by its author. The approach taken regarding speculation is thus: The logical inclusion of probable events that resolve mysteries, while maintaining the themes of the series.

        Advertisement

        Before we begin, let’s briefly revisit the five points of essential lore:

        • Metroids are a genetically-engineered species, created by the Chozo in the prehistory of the games. By the time the first entry begins, Metroid creatures only exist on the planet SR388. At some point long before the games, there was also a Metroid presence on the planet Phaaze.
        Advertisement

        (Metroid IL Return of Samus, Metroid Prime III: Corruption and Metroid Fusion)

        • At least some Chozo possessed a native ability to see into the future.

        (Metroid: Zero Mission and Metroid Prime)

        • The Chozo discovered the living planet Phaaze with their Elysian Research Outpost. We are not told what transpired immediately after this discovery, but something happened that caused at least one Metroid organism to appear on Phaaze. The planet then loaded this creature – along with pieces Chozo-style powersuit armour, into a Phazon seed and launched it towards a heavily populated Chozo planet. This seed impacted Tallon IV and is contained by the Chozo within an impenetrable shield. The mutated superevolved Metroid creature within – clad in crafted power armour – is trapped in the shield until Samus Aran deactivated it thousands of years later.
        Advertisement

        (Metroid Prime EU release, Metroid Prime III: Corruption and Metroid Prime Trilogy)

        • There are living planets in the Metroid Universe. Phaaze is explicitly referred to as being alive, and could interact with a sentient mind – as shown when a Galactic Federation Aurora Unit is implanted. The planet SR388 could be interpreted as having some form of sentience – it shook with apparent anger when its creatures were killed by Samus Aran, and precisely manipulated its oceans to lure the bounty hunter into hostile situations. (Metroid II: Return of Samus and Metroid Prime III: Corruption). The immediate backstory to the first game in the series is the discovery of the planet SR388. The final event of the final game in the chronology is the final destruction of SR388.
        Advertisement

        (Metroid, Metroid II: Return of Samus)

        Referencing

        Each story section includes one or more of the below superscript annotations, to help inform the reader as to where the lore or speculation comes from. A brief key:

        Advertisement

        With all that said, let us begin.

        Part One: The Wars in Heaven

        The Living Planet

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: 3ihard)

        On an unknown planet in the universe, a race of avian humanoids evolved. The species that will come to be known as the Chozo possessed great strength, agility and intelligence. The species is peaceful, and is driven by a social/religious value that nature is sacred. [M1 / MP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        Certain blessed individuals were born with a unique gift – the vague comprehension of events set to take place in the distant future. Driven by these prophecies, the race advanced quickly and became space faring. With abstract predictions of a hostile universe, the Chozo developed powered armour and armaments to defend themselves. Prepared for whatever hostility awaited them, the Chozo explored the stars. [M1 / MP / MP SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Elearia)

        The Chozo discovered that – despite their prophets’ visions of a chaotic and warring universe – the cosmos was enjoying a prolonged period of peace and enlightenment. First contact was made with a number of old and wise races, such as the Ylla, the N’kren, the Bryyonians, the Alimbic and the Luminoth. The species shared their cultures and technology, and gently colonised wild worlds such as Aether, Elysia, and Tallon IV. [MP / MPH / MP2 / MP3]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Slapshoft)

        Peace reigned through the cosmos. The alliance was a great universal renaissance, and lasted for a millennium. [MPH SP / MP2 SP / MP3 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        During this calm, the Chozo prophets continued to receive increasingly severe visions of chaos. They foresaw a universe consumed by war, horrors evolving on distant worlds, and a great toxicity waiting to be unleashed. As the visions became more precise, the species isolated itself from its allies. The Chozo civilisation became intensely driven to fight this unclear threat. [MP / MP3 SP / M2 SP /MF SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: DanilLovesFood)

        The Chozo needed more potent tools to locate this unseen and distant danger. They expanded their SkyTown colony on the gas giant Elysia and remade it into a vast interstellar observatory powered by the planet’s endless storms. The facility was of such scale that an entire species of artificial life became necessary to maintain it. The Chozo created their first species – the mechanical Elysians. [MP3 / MP3 SP]

        Advertisement

        Probes were launched across the universe, and the Elysians and Chozo scrutinised the data. The search took generations, while the planet’s tempestuous atmosphere battered SkyTown, weathering the station faster than the Elysians could maintain it. After countless probe launches, a partial transmission received from a decaying and distant satellite set prophecy in motion. [MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Mechanical-Hand)

        The data received was terrifying. The blue planet registered as an organism, somehow existing as both mineral and flesh. Impossible radiation pulsed from the surface, which overwhelmed the Chozo satellite and rendered it inert. The location of the planet was immediately lost, and only a broad region of space could be established. [MP3]

        Advertisement

        With this find, the Chozo purpose on SkyTown was fulfilled. The race departed the facility, leaving the Elysians to continue their monitoring of the stars. The abandoned race of robots continued to launch satellites to try and rediscover the blue world, hopeful that such a discovery would herald the return of their Chozo creators. The Elysians searched unsuccessfully until Elysia’s endless storms eroded their civilisation into a rusted remnant. [MP3]

        The Chozo reconciled their vague discovery of a blue living planet with their prophecies of toxicity. On this distant world of poison, could creatures have evolved so vicious that they endangered the universe? [MP3 SP]

        Advertisement

        The Invasion of Phaaze

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: 3ihard)

        Finding the exact location of the deadly planet becomes a priority for the Chozo civilisation. A gargantuan ship was assembled on the holy planet of Tallon IV, and dispatched to the dark corner of the universe where the Elysian satellite had been lost. The greatest Chozo warriors, scientists and prophets commenced a crusade for the hostile world, knowing that they would likely never make it back home. During their long journey, they conceive a name for their target: Phaaze. [MP3 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: SesakaTH)

        Generations passed, and the Chozo expedition finally located the blue planet. As they approached, they witnessed the living world as it endlessly pulsed with blue and white energies. There was nothing like this place elsewhere in the universe. [MP3 SP]

        Advertisement

        Their scans confirmed their worst fears – this atmosphere was a bath of radiation and mutation – and evolution had produced horrors. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: SamusMMX)

        For billions of years, Phaaze had mutated and irradiated life that evolved on its surface. The strongest creatures had survived to thrive in an ecosystem of beautiful poison. It was then that the Chozo understood: They had arrived at the home of the most devastating and deranged creatures in the known universe. [MP3 SP]

        Advertisement

        If these monsters were to escape their containment on Phaaze, they would voraciously consume their way through the cosmos. With younger races only centuries away from space travel, the Chozo could not risk them finding this world and releasing its terrors. [MP3 SP]

        The Chozo expedition came to an impasse. The threat of Phaaze’s superpredators had to be neutralised, but severe action against the planet would be sacrilege. The Chozo held life sacred, and refused to destroy the unique living world. [MP3 SP]

        Advertisement

        A dangerous plan was agreed upon. The expedition ship landed on Phaaze, exposing the crew to tremendous radiation. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Adoublea)

        Chozo Warriors in power suits fought the planet’s creatures as they swarmed the ship. The soldiers battled, watching their kin die around them, in a desperate mission to buy time. [MP3 SP]

        Advertisement

        The scientists within the ship began to harness the intense radiation around them, to try and engineer an artificial predator that could neutralise the planet’s superpredators. With access to the unique Phazon mutagen that covered the poisonous world, genetic engineering that should have taken decades was done in days. The Chozo engineered the first Metroid. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Methuselah3000)

        The Metroid creature was unleashed onto the planet, and the radiation caused it to reproduce quickly. The resulting swarm of Metroids began to consume the planet’s monstrosities and established themselves as Phaaze’s apex predator. [MP3 SP]

        Advertisement

        The Chozo mission was complete. The worst creatures were being hunted to extinction, and the Metroids were expected to die from starvation soon after. The cost had been enormous – most of the crew had been killed defending the ship, and the survivors were deathly ill from radiation poisoning. The burnt and damaged ship took off for the long journey home, but the crew soon succumbed to the radiation they had endured. The autopilot took the ship of Chozo bodies home. [MP3 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: 3ihard)

        On Phaaze, the Metroid presence lasted decades as they consumed the planet’s superpredators. The corpses of Chozo warriors were absorbed into the planet, and their battle armour slowly became weathered and scattered. The planet’s slow sentience developed an outrage that seethed under its continents. It had been violated by the Chozo. As the Metroid infestation began to die out, Phaaze developed a very primitive concept of purpose and retribution. [MP3 SP]

        Advertisement

        Phaaze established a vague awareness of concepts it had absorbed from the brains of the Chozo warrior corpses – and the location of two worlds from the Chozo’s memories. As the planet entered its reproductive cycle, it purposely directed two of its seeds towards the planets Tallon IV and Aether. In the seed sent to the Chozo world, Phaaze included one of the last surviving Metroid creatures and some ruined pieces of Chozo armour, intended as a reminder of the crime Phaaze had endured at their hands. The planet sent its second seed to Aether, as the absorbed memories informed the living planet that its inhabitants were friends of the Chozo, and therefore the enemies of Phaaze. [MP 1 / MP 2 SP / MP3 SP]

        The expedition ship – heavily damaged by radiation and lack of maintenance – was guided back to civilisation by an increasingly erratic auto-pilot. After decades it eventually approached the Chozo world of Zebes, and crash-landed onto its surface. The Chozo civilisation attempted to recover data logs from the wreckage with very limited success – they were able to understand the sacrifice that the heroic crew had made, and confirmed the apparent success of the Metroids in neutralising the creatures on the living planet. The Chozo authorities were unable to establish the location of Phaaze, or recover much in the way of scientific data concerning it. [MP3 SP / SM SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Methuselah3000)

        As the Tallon IV seed began its centuries of travelling through space, the lone Metroid within absorbed vast amounts of Phazon and radiation. It became self-aware, and grew in size, intelligence and strength. It used the ruined pieces of Chozo armour to construct itself an exoskeleton, and descended into madness. The exoskeleton failed to protect the creature from the endless radiation, and the Metroid became as exotic as Phaaze’s extinct superpredators: An undying tortured genius. [MP / MP2 / MP3 / MP3 SP]

        Advertisement

        The creature that would come to be known as Metroid Prime resented Phaaze for imprisoning it in the Leviathan. It resented the Chozo for creating and discarding the Metroids. It decided that it would survive, bring order to the chaotic universe that birthed it, and somehow enslave Phaaze to its will. In its solitude, immortal as a consequence of its mutations, Metroid Prime plotted its revenge against the universe. [MP / MP2 / MP3 / MP3 SP]

        The Dark Planet

        With a clear understanding of the danger of living planets, the Chozo authority commenced a search for similar threats. With far more advanced technology than their ancestors had during the Elysian era, the Chozo were unfortunate enough to find a planet of even greater horrors. [MP 3 SP / M2]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: PeaceFistArtist)

        The Chozo detected strange readings coming from a world in a desolate part of the galaxy. The planet had been previously considered so obscure and unimportant that it didn’t have a name, merely catalogued with the codename SR388 and left to its obscurity. A detailed analysis picked up some extremely strange observations; though seemingly mineral, the caverns and liquids beneath the surface shifted with metabolic rhythm – as if the whole planet was somehow a living thing. A ship was dispatched, and the strongest Chozo warriors braved the caverns beneath the surface. [M2 / M2 SP]

        Advertisement

        Few made it back. They told of a cauldron of evil, an environment so hostile and vicious that it had birthed the most terrible things. [M2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: LightningArts)

        Beneath that planet, evolution had been won by an abomination that could steal the flesh, abilities, memories and strengths of all of its prey. The creature was a fusion of energy and plasma that parasitized on life itself. With no word suitable for the nightmare they had discovered, the Chozo simply called it X. If these X-Parasites somehow gained access to the wider universe, there would be no force that could contain them. [M2]

        Advertisement

        The threat had to be dealt with. Remembering the apparent success of the Chozo expedition to Phaaze, a plan was put into action. The Chozo assembled their best and brightest, their strongest and wisest. They carved their way into SR388, and dispatched mechanical creatures to construct secure facilities. Robots and Chozo warriors repressed all instances of the X-Parasite as they found them, but casualties were high. The planet appeared to fight the Chozo at every turn, it drowned the invaders in acid and unleashed ambushes of creatures. The endless swam of X-Parasites gained strength from the corpses around them. [MP3 SP / M2]

        Deep in the planet, a glass laboratory was created, its walls highly resistant to SR388’s acid belly. Here, in dangerous proximity to the X-Parasites, the Chozo scientists began their work. [M2]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        The Chozo tried to recreate the plan of their ancestors – the use of Metroids to pacify superpredators too dangerous to exist. Without access to the same planetary radiation and materials the Phaaze expedition had, progress was slow. As the war against the planet was raging around them, the Chozo scientists were able to engineer Metroids, but not a variant strong enough to overcome the X-Parasites. As more and more Chozo died protecting the laboratory, a different approach was needed. [M2 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Starshadow76)

        The Chozo succeeded in engineering a Metroid Queen, a colossal creature who would lay Metroid Hatchling eggs. When hatched, these resulting Metroids were strong and durable creatures, and finally potent enough to combat the X menace. The Chozo knew that to completely suppress the parasites, the Metroid presence on SR388 had to be permanent. To ensure that the species would not overfeed on the environment and wipe out its food chains, the scientists hardwired an instinct into the Metroid Queen’s feral mind: Only thirty-nine Metroids were to exist on the planet at any one time. This, it was hoped, would keep their numbers high enough to destroy any X re-emergence, but low enough so that they wouldn’t consume the rest of the life on the planet, and starve to death from lack of food. [M2]

        Advertisement

        The scientists assembled a payload of Hatchling Eggs and the surviving Chozo warriors distributed them across the planet. The eggs hatched quickly, and the X-Parasites were immediately overwhelmed by the infant Metroids. The X-Parasites were quickly hunted to near-extinction, with only a few surviving cells entering a state of suspension deep in the planet. [M2 SP / MF]

        The Chozo had won their war, but only just. Most of the warriors and scientists had not survived, and those that were left had to make sure that the X-Parasites had been permanently suppressed. The planet shook with tremors; the earth shifted and acid poured, as if the world was trying to crush the Chozo in their glass laboratory. [M2 SP]

        Advertisement

        The X-Parasites did not return, and the Metroid Queen continued to scream as her glass prison shook. The Chozo didn’t realise it, but her despair was being heard. [M2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Hermax669)

        SR388 had been violated by the Chozo. Though very different to Phaaze, SR388 had its own vague sense of awareness. It perceived the Chozo as a viral infection, and the dead X-Parasites as part of itself. It understood loss, and shook with ancient rage. [MP3 SP / M2 SP / MF SP]

        Advertisement

        It changed itself to change the Metroids. It adopted them to replace the X-Parasites, and quickly killed the weaker breeds. It moved its radioactive minerals closer to their eggs and soon mutated the species. As SR388 had done with X, it did with the Metroids. It made them strong. [M2 SP / MF SP]

        Alpha, Gamma, Zeta and Omega Metroids spawned quickly, and responded to the screams of their Queen. With their bulk and strength, they smashed through the glass laboratory and slaughtered their Chozo creators. The Chozo warriors were hunted down and crushed. [M2]

        Advertisement

        SR388 developed into a new cauldron of hostility. The Metroids served as the apex predator, and the robots of the Chozo decayed into machine madness and prowled the ruins, killing on sight. The Chozo mission to suppress the X-Parasite had been a success, but the planet had gained its revenge. [M2 / M2 SP / MF]

        Part Two: The End of the Renaissance

        The Holy World

        The Chozo had devastated two planets for the good of the universe, and sustained many causalities. The superpredators of Phaaze were extinct and the X-Parasites were permanently suppressed. With the crisis over, the race became consumed with a collective sense of guilt over their necessary actions. The Chozo believed the life of the universe to be sacred, and had to reconcile their aggressive actions with their faith. [MP SP / MP3 SP / M2 / MF]

        Advertisement

        Worse still, their prophets continued to have visions of endless conflict and death. War was coming to the universe, and it seemed that their sins had not saved them. Many began to doubt these visions, and a schism occurred. [MP/ MP3 SP]

        The bulk of the Chozo civilisation retired themselves from galactic affairs, leaving only a few scattered colonies amongst the stars. The race retreated to the holy planet of Tallon IV, to shun their technologies and begin simpler, poetic lives. These Chozo reconnected themselves to the natural world and tried to find a harmony with it. As time went on, the most potent prophets became manic, and tried to warn their fellows of a great poison that was to come. [M1 / MP]

        Advertisement

        These visions were met with increasing dismissal, but the day finally came when the prophets were believed. After eons swimming in the stars, Phaaze’s seed entered the Tallon system. [MP / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Hameed)

        The Leviathan crashed down, and rained poison and death unto the world. The impact survivors watched as their sacred nature succumbed to the mutagens leaking from the seed, and barricaded themselves in their temples as the flora and fauna transformed. Phazon spread beneath the surface of the dying planet, and radiation storms battered the surface. [MP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Source: Riivka)

        The Chozos’ punishment for their sins, and the fulfilment of Phaaze’s wrath, reached biblical proportions. The Chozo of Tallon IV did not get to rest in peace. Their life energies suffered from Phazon disruption, and upon death they became mad ghosts who screamed forever as they were torn in and out of the material world. In this purgatory, the undead immaterial Chozo murdered anyone they could find. [MP / MP3 SP]

        Advertisement

        As their numbers dwindled, the last of the Chozo constructed a great temple above the impact crater. Within this temple, they used what little technology remained to project an energy field around the Leviathan to slow the spread of contagion. As the Chozo civilisation on Tallon IV was extinguished, their dying prophets told of a hero who would one day emerge, to enter the crater and defeat the evil worm within. [MP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Source: Havoc-DM)

        Within the Impact Crater, Metroid Prime remained trapped within the Chozo energy field. In its armour constructed from ancient Chozo power suits, it continued its wait to be unleashed on the universe. [MP / MP3 SP]

        Advertisement

        Dark Echoes

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: 3ihard)

        On the planet Aether, an ancient race of mystics known as the Luminoth received the horrifying data coming from Tallon IV. In distant times, the Luminoth and the Chozo had been steadfast allies – until the Chozo retreat ended their ties. Desperate to assist, the Luminoth began to organise a rescue mission. [MP2 / MP2 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: PugOfDoon)

        A dark transmission was received from Tallon IV. The image showed a screaming, ghostly Chozo figure, flickering in and out of the living universe. In its undead madness, it spoke for its kin. It raged that they would kill anyone who would set foot on their world. The planet was pandemonium, a cursed world on which the dead could not die. As the signal faded, the Luminoth realised that there was no one left alive to rescue. [MP SP / MP2 SP]

        Advertisement

        The Luminoth were receiving strange readings from the devastated planet. A mutagen was spreading, unlike anything they had ever encountered. They scanned the stars for its source, and made a devastating discovery – a mass of the same mutagen was on a collision course with Aether. Phaaze’s second seed had nearly arrived at its destination. [MP2 SP / MP3 SP]

        The people of Aether turned to their technology to save them. Their planet had no native star of its own, and had been implanted millennia ago with a complex energy network that sustained all life. This system was reverently called the Light of Aether, and harnessed the light of the universe in its mechanism. The Luminoth realised that even with this great power, they could not destroy the Phazon Leviathan. A different approach was needed. [MP2 / MP2 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: SesaKath)

        The Luminoth used their great Light to engineer a small pocket universe, a dark lifeless echo of existence. The plan was bold: they would use the Light of Aether to surgically open the fabric of reality in the path of the Phazon seed, and allow it to harmlessly enter the pocket universe. If all went well, they would be saved. [MP2 SP]

        Advertisement

        The day came, and the Leviathan entered Aether’s atmosphere. The Luminoth commenced their great plan. [MP2 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Adriencgd)

        Phaaze’s seed was a sum of living materials beyond Luminoth comprehension. It hit the pocket universe with incalculable force, and a tsunami of exotic energy ruptured space and time. The equipment containing the dark reality lost containment within moments, and the Luminoth were helpless as their creation expanded across the entire planet. A wave of dark energy absorbed creatures, structures and land into the dark universe, and what was once a single planet – was now two. [MP2 / MP2 SP]

        Advertisement

        The Luminoth surveyed the devastation. The Phazon seed was gone – it had indeed collided with the dark universe. Entire continents, with millions of inhabitants, had vanished with it. [MP2 / MP2 SP

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: AzureParagon)

        In the dark universe, a grotesque world was being born. Previous inhabitants of Aether, having been absorbed when containment of the pocket universe was lost, found themselves twisted by the corrosive new reality around them. Most perished, and their flesh fed the strange carnivorous fungi that glowed sickly colours. Some survivors were mutated by the Phazon slowly spreading beneath the surface, and adapted to survive in the hostility. [MP2 SP]

        Advertisement

        Aether and its echo, the Phazon-infested Dark Aether, existed in synchronicity. As the Luminoth tried to rebuild their planet, it took only decades for cracks to form in the ether separating the two realities. As rips in the universe shattered open, Aether became a battlefield. [MP2]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Xxkiragaxx)

        A womb of Phazon mutation and dark energies had birthed a cunning and ferocious horde. The Ing erupted through the cracks between the two worlds, and commenced slaughter. They were fought back by the Luminoth, and a war began between the two parallel worlds. The Ing invaded Aether with regularity, and killed, pillaged and destroyed all that they could find. The Luminoth retaliated and crusaded into Dark Aether in their Light Suits, on suicide missions to exterminate the source of the Ing menace. Both sides suffered colossal casualties as the decades went on. [MP2]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        The war was being lost by the Luminoth. The Ing had exterminated most of their race and had stolen too many vital technologies. With the theft of essential energy components from the Light of Aether power network, they had become a defeated people. [MP2]

        Advertisement

        The Ing had destroyed all of Aether’s ancient ships, and condemned the Luminoth to no escape from their doomed world. With no other choice, the survivors sealed themselves in an inner sanctum, and entered a state of suspended animation. One custodian, U-Mos, volunteered to be their guardian. As Aether became weaker and weaker, the Luminoth waited for someone to save them. They would wait a very long time. [MP2]

        The Sacrifice of the Alimbics

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Kihunter)

        As the Chozo and the Luminoth fell, so too did other ancient races. In a distant part of the universe, the Alimbics were a militaristic society that maintained peace in their galactic cluster. Their order was shattered when a murderous entity, originating from someplace beyond the understood universe, plummeted into one of their worlds. The creature emerged from the devastation as a gaseous entity, and assumed an Alimbic-styled body to begin its onslaught. [MPH]

        Advertisement

        This alien juggernaut was named Gorea by the Alimbic race, and they soon understood it brought only death. Gorea killed every Alimbic it could find, and destroyed everything in its path. Planet after planet fell to Gorea, and the Alimbics realised the creature would never stop. [MPH]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Sesakath)

        The Alimbics performed an act of supreme sacrifice. They combined the mental energies of their entire race to forge a prison for Gorea. The psychic prison held it bound, and it was transplanted into an organic vessel called The Oubliette. The vessel was launched into the void outside of the universe, a course that would keep its indestructible prisoner in exile forever. The systems of the prison ship were tasked to scan the every molecule of the imprisoned Gorea, and devise an Omega weapon that could be used to kill it. [MPH / MPH SP]

        Advertisement

        The mental energies expelled in this plan consumed the physical bodies of the entire Alimbic race. They vanished from the universe in an instant. Their sacrifice protected all life in the cosmos from Gorea’s murderous rampage. [MPH]

        The War of Bryyo

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Sesakath)

        As the old races of the universe died around them, the lizard people of Bryyo faced their own challenges. The Bryyonians were an advanced, space-faring race who had learned much from their Chozo allies. Their society was a deeply polarised one, with tensions eternal between the scientific and religious factions.[MP3]

        Advertisement

        Over the previous centuries, the scientific agenda had dominated, with space travel proving beneficial and enlightening. As the Chozo, Luminoth and Alimbics faced extinction, the religious Bryyonians believed more than ever that the universe was a hostile place, and became desperate to stop their scientific counterparts. [MP3]

        A great war exploded across Bryyo. By its end, the scholars had been wiped out and the survivors of both sides had regressed to a feral existence. The race devolved into animals, wandering around ruins that they no longer understood. Language vanished and strength ruled. Anyone who landed on Bryyo was meat, to be killed and eaten. [MP3]

        Advertisement

        The Little Rainy Planet

        The onslaught of vengeances, conquerors, poisons and politics destroyed the old races. The Alimbics had lost their flesh, while the Bryyonians had lost their souls. The Luminoth had retreated into stasis, and the Chozo of Tallon IV had been condemned to a living death. [MP / MPH / MP2 / MP3]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Kaiquesilva)

        On a small, rainy planet called Zebes, the last known Chozo colony had watched the stars with impotence. This small settlement of the nearly-extinct avian race witnessed the end of the great universal renaissance, and the slow beginning of a new chapter in galactic history. Gradually, the younger races were launching their first satellites into space. In time, new empires would rise to take the place of the old. [M1 / M1 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: 3ihard)

        Zebes prophets’ saw the visions the Chozo had always endured: great wars, spreading poison and death. And suddenly, something bold was foreseen. [M1 SP / MP3 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Fddt)

        A great hunter, clad in orange, red and green. The Chozo glimpsed a future hero, alone in the darkness beneath worlds, fighting so that good could survive evil. They saw her curing poisoned planets, and ending galactic wars. They saw the universe’s one chance to survive its apocalyptic future. They saw the only one who could defy prophecy. [M1 / MP3 SP]

        Advertisement

        And they saw her wearing Chozo armour. [M1]

        Part Three: The New Empires

        The Humans

        On the planet Earth, the human race had finally developed a ship capable of leaving their solar system. A brave crew ventured into the universe to learn whether life existed elsewhere. Their discoveries fundamentally changed the human condition. On planet after planet, they found ruined tombs and cities, guarded by weathered statues of dead races. Most significant of all, they found technology. [M1 SP]

        Advertisement

        The humans reverse engineered their salvage, and advanced with pace. Within another century, faster-than-light ships explored the stars, and colonies transformed hostile worlds into homes. Peaceful relations formed between other younger races, and a great Galactic Federation was founded. [M1 SP]

        The Space Pirates

        In a less hospitable region of space, a cabal of battered races joined their forces to survive. On planets where acid rain burned flesh and magma flowed, the alliance expanded into a hardened space empire. They ventured into nearby systems and took what they needed from anyone they could reach. They found the ruins of the old races and ransacked the ancient technologies within. They immersed themselves in science and unlocked the secrets of their finds. Within decades, they had advanced their spread with stronger and faster ships. The creatures enhanced themselves, rewriting their genetics and integrating mechanisms beneath their flesh. They were unique: a cybernetic race of furious murderers with a skill for patient scientific process. As more planets were invaded, their conquered civilisations were conscripted by force. [M1 SP / MP / MP3]

        Advertisement

        The inevitable moment came when their Empire reached the borders of the vast Galactic Federation. [M1 SP / MP / MP3]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Mr-Corr)

        First contact was brief and furious. On that day, the warning went out to all the worlds of the Federation: Beware the Space Pirates. Though no state of war was officially declared, the empires attacked each other on sight. The Galactic Federation was large enough to repress any meaningful incursions into their space. [M1 SP / MP SP / MP3 SP / SM SP]

        Advertisement

        The Massacre of Two Families

        The Galactic Federation discovered the last Chozo Colony on Zebes. The tired, ancient avians welcomed the humans and shared with them wisdom and knowledge. They offered the Galactic Federation new sciences, and taught them how to make organic computers. The Federation studied the Chozo’s own central processing unit, an engineered brain that mothered over their colony, and left with plans to assemble their own variants. On the nearest habitable planet of K-2L, a colony was established. [M1 / MP3 SP]

        Advertisement

        On this world, the human Samus Aran was born. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Methuselah3000)

        Barely out of infancy, the young Samus witnessed her family die. A Space Pirate raiding party overwhelmed her colony and murdered everyone she ever knew. By staying silent while surrounded by horror, Samus survived as the Pirates ransacked the settlement and left. [M1]

        Advertisement

        The Chozo colony on Zebes received K-2L’s automated distress signal. In an ancient dusty ship, they reached the planet and found Samus to be the only survivor of the massacre. The child was brought to Zebes, and the Chozo deliberated. Should she be returned to her own kind, or allowed to stay? [M1]

        Across the colony, the Prophets experienced a simultaneous moment of clarity. They understood immediately that they had found their prophesised hero. The young girl was their inheritor, and would grow strong. She would learn all she could from them, and take their strongest technologies into the universe. She would be the hero against the oncoming storm. [M1 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: R3dFiVe)

        Samus Aran reached maturity amongst the Chozo. She was trained in the combat arts of the great extinct races. She was infused with Chozo genetic material so she could employ their technologies. She was educated to be a scientist, an explorer, and a tactician. Everything that was good about the Chozo civilisation was allowed to live on in Samus. [M1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Pyra)

        Samus became an adult, and the Chozo presented her with their greatest works: a toughened power suit and an agile spacecraft, both more potent than anything their race had ever made. The Chozo leader, decaying and blind, told Samus it was time for her to find her destiny in the universe. Samus Aran departed for the stars, and years pass. [M1 / M1 SP]

        Advertisement

        As Samus tried to reconnect with her heritage on Earth, the last Chozo prophets on Zebes received a final vision: The Space Pirates were coming for them. It was time for the last Chozo to be extinguished from the universe. [M1 SP]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Phobos-Romulus)

        The Chozo hid their technologies throughout the planet, in places that they were certain Samus would find them. They concealed a second Power Suit within the walls of their holy temple, having foreseen that Samus may require it in the future. They then returned to the surface to await the inevitable. [M1 SP]

        Advertisement

        The Space Pirates invaded in force, and murdered Samus Aran’s second family. The Chozo became extinct. [M1 / MP SP]

        The Mother Brain

        Space Pirate scientists arrived shortly after the carnage and focused their attention on the legendary Chozo organic central processing unit. They rewrote its benign programming and injected stimulants into its flesh. They enabled it to form an artificial intelligence obsessed with strategy and conquest. They drove its computational potential towards absolute advancement of the Space Pirate Empire. [M1 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Jaagup)

        The results went beyond High Command’s most optimistic projections. The Space Pirates had created a leader, a desperately needed figure to unite their fragmented empire. They had created their Mother Brain. The great Space Pirate generals Ridley and Kraid arrived at Zebes, ready to pay tribute to their new master and to plan for the future. Mother Brain delivered to the Space Pirates knowledge and power. She told them of a world referenced in her oldest Chozo databanks, a planet bathed in a mutagenic poison waiting to be farmed. She instructed High Command to prepare an armada of ships and invade the planet Tallon IV. [M1 / MP SP]

        Advertisement

        The order was followed immediately, and the High Command discovered a world deranged by contagion. Beneath its surface, endless pools of Phazon waited to be weaponised, and a great mining operation began. Mother Brain received data from their readings on the planet; even after thousands of years, the source of the Phazon was still contained in the Chozo force field. She scrutinised her records further, and was unable to ascertain any method of breaching the barrier. The Space Pirates could retrieve the Phazon, but were denied access to its source. [MP]

        The Metroids

        A perfect storm brewed. As the Space Pirates gained access to the most potent mutagen in the universe, the Galactic Federation made an equally eventful discovery: They found the dark planet SR388. [M1 / M2]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Fireborn Form)

        A Galactic Federation survey team studied the surface, and soon encountered a gelatinous creature that swam through air. The alien defied gravity and physics as it phased through dense rock with ease. It perceived the survey team, and made a few curious chirps in their direction. It then suddenly changed temperament, aggressively charging to latch itself onto the skull of one of the party. The victim died in agony as the Metroid fed on all the energy within, and could not be removed until its prey had been reduced to a dried husk of collapsing matter. The young Metroid had just killed, in a way that science could not explain. [M1 SP / M2 SP]

        Advertisement

        With effort and casualties, the scientists contained a few infant specimens of the Metroid creatures, and left the planet without further incident. [M1]

        As their vessel went back to the stars, SR388 was aware their withdrawal. It harboured a great contempt for the invaders, an endless hate fuelled by the impotence it endured centuries ago when the Chozo had committed their great invasion. The living planet had spent centuries honing the Metroids into perfect killers, and knew the devastation they could cause upon maturity. The planet had intentionally allowed the humans to take a few Metroids away so that the creatures could grow up and kill anyone out amongst the stars who ever thought of returning. SR388 took any opportunity to gain revenge against an outside universe that refused to leave it alone. [M2 SP]

        Advertisement

        As the scientists began to broadcast their findings back to the Galactic Federation, Mother Brain intercepted the transmission. She cross-referenced their data with notes buried in the Chozo’s ancient fragmented records. She deduced that the Metroids were a form of genetically engineered predator of incredible power, created by the Chozo for an unknown purpose. Mother Brain ordered High Command to get the creatures to her by any means necessary. [M1 SP]

        The Space Pirates overran the Galactic Federation vessel and stole the Metroid creatures. They divided their prize: some were sent to their nearest Homeworld; others were sent to the Tallon IV outpost; and the most potent were delivered straight to Zebes for the experiments of Mother Brain. [M1 / MP / MP3]

        Advertisement

        With the arrival of the first Phazon samples from Tallon IV, the exotic substance allowed the Space Pirates to slowly produce stable cloned Metroids across their breeding sites. [M1 SP / MP SP]

        The Revenge of Samus Aran

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Ojanpohja)

        In her first mission as a Bounty Hunter, Samus Arran was commissioned by the Galactic Federation to neutralise the stolen Metroids. Through careful investigation, Samus discovered that the Pirates are operating from Zebes – her home. She concluded that the Space Pirates had murdered her second family, as they had done with her first. They have took from her everyone she ever loved, and destroyed her two worlds. [M1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Stuart Hughe)

        Samus stormed Zebes and killed everyone in her path. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Immarart)

        As her defences were breached, Mother Brain unleashed the great generals Ridley and Kraid. Both were killled, and, desperate to stop the intruder, Mother Brain released the Metroids. Samus Aran exterminated the creatures, and invaded the inner sanctum. [M1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Twigs)

        Samus confronted the malevolent Mother Brain and blasted apart her body. A power overload was caused, and the Tourian facility shook itself apart. Samus evacuated to her ship and tried to leave Zebes, but a Space Pirate battleship in orbit registered her ascent and opened fire. Samus’ gunship plummeted back towards the Zebes and impacted Chozodia, her former home. [M1]

        Advertisement

        Extremely lucky to be alive, Samus crawled out of the remains of her destroyed power suit, and fled as Space Pirate forces stormed the area. Samus hid, crawled and ran to find sanctuary in the deepest part of the Chozo’s most revered temple. [M1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Eyes5)

        Samus found herself surrounded with murals of the dead Chozo, and accepted she was alone in the universe. Overcoming despair, she solved the trials of the Chozodian temple and a concealed power suit was revealed to her. This shining armour was even more potent than the one she had just lost, and was able to integrate the most exotic Chozo technologies. Samus realised the greater meaning of her find; the Chozo had left her gifts for her in places they had foreseen she would traverse. Her adopted family continued to protect her long after their deaths, and she would find their statues cradling survival equipment in the darkest corners of the cosmos. [M1 / MP / MP3 / M2 / SM]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Imachinivid)

        With her new armaments, Samus cleansed the Space Pirate presence from Zebes. She came to be known as “The Hunter”, and the Space Pirates learned that they will always be hunted down for what they did to her families. They fled the planet in terror. [M1 / MP / MP2 / MP3]

        Advertisement

        Tallon IV

        As years passed, Samus Aran accepted further missions from the Galactic Federation. The bounty earned funded her personal vendetta against the Space Pirates. She improved her armaments, paid for black market information and stormed their outposts. Samus showed her enemies no mercy, and became the feared nemesis of their entire civilisation. With the income from her Federation services, Samus had soon amassed enough money to buy the most secret information regarding the Space Pirates: the coordinates of their stronghold on an old forgotten planet called Tallon IV. [MP1 SP]

        Advertisement

        Samus guided her ship into the Tallon system and investigated an orbiting space station. She discovered a failed genetic engineering facility whose Space Pirate crew was murdered when they lost control of their own creations. Samus fought her way through the ferocious beasts scattered within, and discovered a half-insane cyborg recreation of the Space Pirate general Ridley. As the station began to collapse, the biomechanical dragon fled to the world below, and Samus pursued. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: Lightningarts)

        Samus lost Ridley in the planet’s stormy atmosphere, and elected to land in a nearby jungle to conceal her presence from the Pirate ground forces. Exploring the surroundings, Samus discovered that the planet was once home to the bulk of the extinct Chozo civilisation. In a great temple Samus studied poetic murals that told of the Phazon comet that had struck their world. The scribblings informed her of a creature trapped deep in the comet that they referred to as “The Worm,” and of the powerful shield they erected to prevent its escape. Samus read their last prophecy; that a hero would traverse fire and ice, jungle and cave, and find twelve sacred keys that would deactivate the barrier and allow passage to the Impact Crater. This saviour from the stars would bring down the ancient shield, and destroy the worm that infected their planet. [MP1]

        Advertisement

        She continued her exploration, and battled ferocious flora and fauna. The Hunter came to understand that the Space Pirates had established a complex military installation that descended far below the surface. [MP1]

        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes
        Advertisement

        (Artist: R-Sraven)

        Samus hunted the Pirates and accessed their computer logs. The Empire had found quantities of an intensely potent mutagen called Phazon. Laboratories across the outpost experimented with the substance, and in a short space of time they had created prototypes for the next generation of their races: powerful Phazon-fuelled juggernauts. Should these advances continue, Samus knew that the Space Pirates would be able to conquer the Galactic Federation. [MP1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Greenstranger)

        In the most secure laboratory, Samus made a devastating discovery. The Space Pirates had used Phazon to create an army of stable clone Metroids and lost containment. The Metroid creatures were roaming the caverns deep in the planet, reproducing and mutating as the Phazon influenced their physiology. [MP1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Ohimseeinstars)

        Samus’ final discovery was the most horrific. The powerful, poisonous Phazon was not a rare material on Tallon IV. Despite the Chozo shield containing the Impact Crater, the substance had spread and consumed the world inside-out. The core of the planet presented the Space Pirates with a vast supply of Phazon, enough to fuel their conquest of the stars. [MP1]

        Advertisement

        Samus destroyed the mining facilities and laboratories, and reconstructed the twelve parts of the ancient Chozo cipher. She destroyed living weapons such as the Thardus experiment, and annihilated the prototype Omega Pirate. She overcame corrupted Metroids, and banished the tormented Chozo ghosts from the living world. She fought the mad Meta Ridley, and on his demise deactivated the Chozo containment shield. As prophesised, Samus Aran entered the Impact Crater. [MP1]

        The Worm

        Samus Aran had opened Metroid Prime’s cage, and had no understanding of what she was about to unleash on the universe. The creature had been imprisoned in a different era, and had spent eons being tortuously transformed by Phazon into an undying mad genius. [MP1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Chrysaetos-Pteron)

        Samus and the ancient Metroid battled, and the bounty hunter shattered the creature’s metal armour. By channelling the surrounding Phazon deposits into a supercharged energy beam, Samus was able to devastate Metroid Prime’s gelatinous body. After a tremendous battle, the old creature began to collapse on itself. [MP1]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Sabretoontigers)

        Seemingly dying, Metroid Prime lashed out, grabbing a layer of material from Samus Aran’s armour. The creature melted into a pool of Phazon particles, and the bounty hunter evacuated the Impact Crater. [MP1]

        Advertisement

        Samus Aran had seemingly succeeded in her mission. The surviving Space Pirates abandoned their devastated facilities and hastily evacuated the planet. With the defeat of Metroid Prime, the Phazon contagion was slowly stopping its spread. The tormented Chozo spirits that had been bound to the planet were finally able to achieve their rest. Leaving the world to recover from its devastation, Samus Aran headed back to the stars. [MP1 / MP1 SP]

        The Dark Hunter

        Metroid Prime’s exposure to millennia of Phazon had given the creature extremely exotic abilities, the most potent being its durability – to recreate itself after nearly any level of destruction. As it had collapsed on itself, the essence of Metroid Prime craved the strength and adaptability present in Samus Aran. In the Talon IV impact crater, Metroid Prime recreated itself as a dark copy of the woman who had defeated it. [MP 1 / MP2 / MP3 SP]

        Advertisement
        Illustration for article titled The Spectacular Story Of emMetroid/em, One Of Gamings Richest Universes

        (Artist: Imachinivid)

        Dark Samus clawed its way out of the Impact Crater. It departed Tallon IV to spread its venom across the stars, and to sow the seeds of a great war. [MP1 / MP2 / MP3]

        Advertisement

        Click here for the second half of this epic story.

        Mama Robotnik is a video game historian living somewhere in the British Empire. He specialises in unearthing lost gaming media, but also enjoys a good long essay about his favourite games every now and then. He drinks a lot of tea, and has a horrendously naughty black and white cat called Blossom. If you would like to contact him, he responds to his private messages over at NeoGAF.

        Share This Story

        Get our newsletter

        More from Kotaku

        Sources: Despite Huge Sales, Borderlands 3 Developers Are Getting Stiffed On Bonuses

        Let's Rank The Total War Games, From Best To Worst

        Cosplayers Can't Go To Cons, So They Had One On The Internet [UPDATE]

        Professional Fashion Photographer Now Taking Snaps In Animal Crossing

        ================================================ FILE: test/test-pages/lazy-image-3/expected-metadata.json ================================================ { "title": "Lazy Load with Alt includes jpg/png/webp extensions", "byline": null, "dir": null, "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/lazy-image-3/expected.html ================================================

        Test Case 1

        performance.jpg

        Test Case 2

        performance.jpg
        ================================================ FILE: test/test-pages/lazy-image-3/source.html ================================================ Lazy Load with Alt includes jpg/png/webp extensions

        Test Case 1

        performance.jpg

        Test Case 2

        performance.jpg
        ================================================ FILE: test/test-pages/lemonde-1/expected-metadata.json ================================================ { "title": "Le projet de loi sur le renseignement massivement approuvé à l'Assemblée", "byline": "Martin Untersinger (avec Damien Leloup et Morgane Tual)", "dir": null, "lang": "fr", "excerpt": "Largement approuvé par les députés, le texte sera désormais examiné par le Sénat, puis le Conseil constitutionnel.", "siteName": "Le Monde.fr", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/lemonde-1/expected.html ================================================

        Les députés ont, sans surprise, adopté à une large majorité (438 contre 86 et 42 abstentions) le projet de loi sur le renseignement défendu par le gouvernement lors d’un vote solennel, mardi 5 mai. Il sera désormais examiné par le Sénat, puis le Conseil constitutionnel, prochainement saisi par 75 députés. Dans un souci d'apaisement, François Hollande avait annoncé par avance qu'il saisirait les Sages.

        Revivez le direct du vote à l’Assemblée avec vos questions.

        Ont voté contre : 10 députés socialistes (sur 288), 35 UMP (sur 198), 11 écologistes (sur 18), 11 UDI (sur 30), 12 députés Front de gauche (sur 15) et 7 non-inscrits (sur 9). Le détail est disponible sur le site de l'Assemblée nationale.

        Parmi les députés ayan voté contre figurent notamment des opposants de la première heure, comme l'UMP Laure de la Raudière ou l'écologiste Sergio Coronado, mais aussi quelques poids lourds de l'opposition comme Patrick Devedjian ou Claude Goasguen. A gauche, on trouve parmi les quelque opposants au texte Aurélie Filipetti. Christian Paul, qui avait été très actif lors d'autres débats sur les libertés numériques, s'est abstenu.

        Pouria Amirshahi, député socialiste des Français de l'étranger qui a également voté contre, a annoncé qu'il transmettrait un « mémorandum argumenté » au Conseil constitutionnel et demanderait à se faire auditionner sur le projet de loi. D'autres députés ont prévu de faire la même démarche.

        Ce texte, fortement décrié par la société civile pour son manque de contre-pouvoir et le caractère intrusif des techniques qu’il autorise, entend donner un cadre aux pratiques des services de renseignement, rendant légales certaines pratiques qui, jusqu’à présent, ne l’étaient pas.

        Retour sur ses principales dispositions, après son passage en commission des lois et après le débat en séance publique.

        Définition des objectifs des services

        Le projet de loi énonce les domaines que peuvent invoquer les services pour justifier leur surveillance. Il s’agit notamment, de manière attendue, de « l’indépendance nationale, de l’intégrité du territoire et de la défense nationale » et de « la prévention du terrorisme », mais également des « intérêts majeurs de la politique étrangère », ainsi que de la « prévention des atteintes à la forme républicaine des institutions » et de « la criminalité et de la délinquance organisées ». Des formulations parfois larges qui inquiètent les opposants au texte qui craignent qu’elles puissent permettre de surveiller des activistes ou des manifestants.

        La Commission de contrôle

        Le contrôle de cette surveillance sera confié à une nouvelle autorité administrative indépendante, la Commission nationale de contrôle des techniques de renseignement (CNCTR), composée de six magistrats du Conseil d’Etat et de la Cour de cassation, de trois députés et trois sénateurs de la majorité et de l’opposition, et d’un expert technique. Elle remplacera l’actuelle Commission nationale de contrôle des interceptions de sécurité (CNCIS).

        Elle délivrera son avis, sauf cas d’urgence, avant toute opération de surveillance ciblée. Deux types urgences sont prévus par la loi : d’un côté une « urgence absolue », pour laquelle un agent pourra se passer de l’avis de la CNCTR mais pas de l’autorisation du premier ministre. De l’autre, une urgence opérationnelle extrêmement limitée, notamment en termes de techniques, à l’initiative du chef du service de renseignement, qui se passe de l’avis de la CNCTR. Ces cas d’urgence ne justifieront pas l’intrusion d’un domicile ni la surveillance d’un journaliste, un parlementaire ou un avocat. Dans ces cas, la procédure classique devra s’appliquer.

        L’avis de la CNCTR ne sera pas contraignant, mais cette commission pourra saisir le Conseil d’Etat si elle estime que la loi n’est pas respectée et elle disposera de pouvoirs d’enquête. Ce recours juridictionnel est une nouveauté dans le monde du renseignement.

        Les « boîtes noires »

        Une des dispositions les plus contestées de ce projet de loi prévoit de pouvoir contraindre les fournisseurs d’accès à Internet (FAI) à « détecter une menace terroriste sur la base d’un traitement automatisé ». Ce dispositif – autorisé par le premier ministre par tranche de quatre mois – permettrait de détecter, en temps réel ou quasi réel, les personnes ayant une activité en ligne typique de « schémas » utilisés par les terroristes pour transmettre des informations.

        En pratique, les services de renseignement pourraient installer chez les FAI une « boîte noire » surveillant le trafic. Le contenu des communications – qui resterait « anonyme » – ne serait pas surveillé, mais uniquement les métadonnées : origine ou destinataire d’un message, adresse IP d’un site visité, durée de la conversation ou de la connexion… Ces données ne seraient pas conservées.

        La Commission nationale informatique et libertés (CNIL), qui critique fortement cette disposition. La CNIL soulève notamment que l’anonymat de ces données est très relatif, puisqu’il peut être levé.

        Lire aussi : Les critiques de la CNIL contre le projet de loi sur le renseignement

        Le dispositif introduit une forme de « pêche au chalut » – un brassage très large des données des Français à la recherche de quelques individus. Le gouvernement se défend de toute similarité avec les dispositifs mis en place par la NSA américaine, arguant notamment que les données ne seront pas conservées et que cette activité sera contrôlée par une toute nouvelle commission aux moyens largement renforcés. Il s’agit cependant d’un dispositif très large, puisqu’il concernera tous les fournisseurs d’accès à Internet, et donc tous les internautes français.

        L’élargissement de la surveillance électronique pour détecter les « futurs » terroristes

        La surveillance des métadonnées sera aussi utilisée pour tenter de détecter de nouveaux profils de terroristes potentiels, prévoit le projet de loi. Le gouvernement considère qu’il s’agit d’une manière efficace de détecter les profils qui passent aujourd’hui « entre les mailles du filet », par exemple des personnes parties en Syrie ou en Irak sans qu’aucune activité suspecte n’ait été décelée avant leur départ.

        Pour repérer ces personnes, la loi permettra d’étendre la surveillance électronique à toutes les personnes en contact avec des personnes déjà suspectées. En analysant leurs contacts, la fréquence de ces derniers et les modes de communication, les services de renseignement espèrent pouvoir détecter ces nouveaux profils en amont.

        De nouveaux outils et méthodes de collecte

        Les services pourront également procéder, après un avis de la CNCTR, à la pose de micros dans une pièce ou de mouchards sur un objet (voiture par exemple), ou à l’intérieur d’un ordinateur. L’utilisation des IMSI-catchers (fausses antennes qui permettent d’intercepter des conversations téléphoniques) est également légalisée, pour les services de renseignement, dans certains cas. Le nombre maximal de ces appareils sera fixé par arrêté du premier ministre après l’avis de la CNCTR.

        Lire : Que sont les IMSI-catchers, ces valises qui espionnent les téléphones portables ?

        La loi introduit également des mesures de surveillance internationale : concrètement, les procédures de contrôle seront allégées lorsqu’un des « bouts » de la communication sera situé à l’étranger (concrètement, un Français qui parle avec un individu situé à l’étranger). Cependant, comme l’a souligné l’Arcep (l’Autorité de régulation des communications électroniques et des postes), sollicitée pour le versant technique de cette mesure, il est parfois difficile de s’assurer qu’une communication, même passant par l’étranger, ne concerne pas deux Français.

        Un nouveau fichier

        La loi crée un fichier judiciaire national automatisé des auteurs d’infractions terroristes (Fijait), dont les données pourront être conservées pendant vingt ans.

        Ce fichier concerne les personnes ayant été condamnées, même si une procédure d’appel est en cours. Les mineurs pourront aussi être inscrits dans ce fichier et leurs données conservées jusqu’à dix ans. L’inscription ne sera pas automatique et se fera sur décision judiciaire. Certaines mises en examen pourront aussi apparaître sur ce fichier. En cas de non-lieu, relaxe, acquittement, amnistie ou réhabilitation, ces informations seront effacées.

        Renseignement pénitentiaire

        Le renseignement pénitentiaire pourra, dans des conditions qui seront fixées par décret, profiter des techniques que légalise le projet de loi pour les services de renseignement. La ministre de la justice, Christiane Taubira, était défavorable à cette disposition, soutenue par le rapporteur du texte, la droite et une partie des députés de gauche. Pour la ministre, cette innovation va dénaturer le renseignement pénitentiaire et le transformer en véritable service de renseignement.

        Conservation des données

        La CNIL a fait part à plusieurs reprises de sa volonté d’exercer sa mission de contrôle sur les fichiers liés au renseignement, qui seront alimentés par ces collectes. Ces fichiers sont aujourd’hui exclus du périmètre d’action de la CNIL.

        La durée de conservation des données collectées – et l’adaptation de cette durée à la technique employée – a par ailleurs été inscrite dans la loi, contrairement au projet initial du gouvernement qui entendait fixer ces limites par décret. Elle pourra aller jusqu’à cinq ans dans le cas des données de connexion.

        Un dispositif pour les lanceurs d’alerte

        La loi prévoit également une forme de protection pour les agents qui seraient témoins de surveillance illégale. Ces lanceurs d’alerte pourraient solliciter la CNCTR, voire le premier ministre, et leur fournir toutes les pièces utiles. La CNCTR pourra ensuite aviser le procureur de la République et solliciter la Commission consultative du secret de la défense nationale afin que cette dernière « donne au premier ministre son avis sur la possibilité de déclassifier tout ou partie de ces éléments ». Aucune mesure de rétorsion ne pourra viser l’agent qui aurait dénoncé des actes potentiellement illégaux.

        ================================================ FILE: test/test-pages/lemonde-1/source.html ================================================ Le projet de loi sur le renseignement massivement approuvé à l'Assemblée
        Pixels

        Le projet de loi sur le renseignement massivement approuvé à l'Assemblée

        Le Monde | • Mis à jour le | Par

        Les députés ont, sans surprise, adopté à une large majorité (438 contre 86 et 42 abstentions) le projet de loi sur le renseignement défendu par le gouvernement lors d’un vote solennel, mardi 5 mai. Il sera désormais examiné par le Sénat, puis le Conseil constitutionnel, prochainement saisi par 75 députés. Dans un souci d'apaisement, François Hollande avait annoncé par avance qu'il saisirait les Sages.

        Revivez le direct du vote à l’Assemblée avec vos questions.

        Ont voté contre : 10 députés socialistes (sur 288), 35 UMP (sur 198), 11 écologistes (sur 18), 11 UDI (sur 30), 12 députés Front de gauche (sur 15) et 7 non-inscrits (sur 9). Le détail est disponible sur le site de l'Assemblée nationale.

        Parmi les députés ayan voté contre figurent notamment des opposants de la première heure, comme l'UMP Laure de la Raudière ou l'écologiste Sergio Coronado, mais aussi quelques poids lourds de l'opposition comme Patrick Devedjian ou Claude Goasguen. A gauche, on trouve parmi les quelque opposants au texte Aurélie Filipetti. Christian Paul, qui avait été très actif lors d'autres débats sur les libertés numériques, s'est abstenu.

        Pouria Amirshahi, député socialiste des Français de l'étranger qui a également voté contre, a annoncé qu'il transmettrait un « mémorandum argumenté » au Conseil constitutionnel et demanderait à se faire auditionner sur le projet de loi. D'autres députés ont prévu de faire la même démarche.

        Ce texte, fortement décrié par la société civile pour son manque de contre-pouvoir et le caractère intrusif des techniques qu’il autorise, entend donner un cadre aux pratiques des services de renseignement, rendant légales certaines pratiques qui, jusqu’à présent, ne l’étaient pas.

        Retour sur ses principales dispositions, après son passage en commission des lois et après le débat en séance publique.

        Définition des objectifs des services

        Le projet de loi énonce les domaines que peuvent invoquer les services pour justifier leur surveillance. Il s’agit notamment, de manière attendue, de « l’indépendance nationale, de l’intégrité du territoire et de la défense nationale » et de « la prévention du terrorisme », mais également des « intérêts majeurs de la politique étrangère », ainsi que de la « prévention des atteintes à la forme républicaine des institutions » et de « la criminalité et de la délinquance organisées ». Des formulations parfois larges qui inquiètent les opposants au texte qui craignent qu’elles puissent permettre de surveiller des activistes ou des manifestants.

        La Commission de contrôle

        Le contrôle de cette surveillance sera confié à une nouvelle autorité administrative indépendante, la Commission nationale de contrôle des techniques de renseignement (CNCTR), composée de six magistrats du Conseil d’Etat et de la Cour de cassation, de trois députés et trois sénateurs de la majorité et de l’opposition, et d’un expert technique. Elle remplacera l’actuelle Commission nationale de contrôle des interceptions de sécurité (CNCIS).

        Elle délivrera son avis, sauf cas d’urgence, avant toute opération de surveillance ciblée. Deux types urgences sont prévus par la loi : d’un côté une « urgence absolue », pour laquelle un agent pourra se passer de l’avis de la CNCTR mais pas de l’autorisation du premier ministre. De l’autre, une urgence opérationnelle extrêmement limitée, notamment en termes de techniques, à l’initiative du chef du service de renseignement, qui se passe de l’avis de la CNCTR. Ces cas d’urgence ne justifieront pas l’intrusion d’un domicile ni la surveillance d’un journaliste, un parlementaire ou un avocat. Dans ces cas, la procédure classique devra s’appliquer.

        L’avis de la CNCTR ne sera pas contraignant, mais cette commission pourra saisir le Conseil d’Etat si elle estime que la loi n’est pas respectée et elle disposera de pouvoirs d’enquête. Ce recours juridictionnel est une nouveauté dans le monde du renseignement.

        Les « boîtes noires »

        Une des dispositions les plus contestées de ce projet de loi prévoit de pouvoir contraindre les fournisseurs d’accès à Internet (FAI) à « détecter une menace terroriste sur la base d’un traitement automatisé ». Ce dispositif – autorisé par le premier ministre par tranche de quatre mois – permettrait de détecter, en temps réel ou quasi réel, les personnes ayant une activité en ligne typique de « schémas » utilisés par les terroristes pour transmettre des informations.

        En pratique, les services de renseignement pourraient installer chez les FAI une « boîte noire » surveillant le trafic. Le contenu des communications – qui resterait « anonyme » – ne serait pas surveillé, mais uniquement les métadonnées : origine ou destinataire d’un message, adresse IP d’un site visité, durée de la conversation ou de la connexion… Ces données ne seraient pas conservées.

        La Commission nationale informatique et libertés (CNIL), qui critique fortement cette disposition. La CNIL soulève notamment que l’anonymat de ces données est très relatif, puisqu’il peut être levé.

        Lire aussi : Les critiques de la CNIL contre le projet de loi sur le renseignement

        Le dispositif introduit une forme de « pêche au chalut » – un brassage très large des données des Français à la recherche de quelques individus. Le gouvernement se défend de toute similarité avec les dispositifs mis en place par la NSA américaine, arguant notamment que les données ne seront pas conservées et que cette activité sera contrôlée par une toute nouvelle commission aux moyens largement renforcés. Il s’agit cependant d’un dispositif très large, puisqu’il concernera tous les fournisseurs d’accès à Internet, et donc tous les internautes français.

        L’élargissement de la surveillance électronique pour détecter les « futurs » terroristes

        La surveillance des métadonnées sera aussi utilisée pour tenter de détecter de nouveaux profils de terroristes potentiels, prévoit le projet de loi. Le gouvernement considère qu’il s’agit d’une manière efficace de détecter les profils qui passent aujourd’hui « entre les mailles du filet », par exemple des personnes parties en Syrie ou en Irak sans qu’aucune activité suspecte n’ait été décelée avant leur départ.

        Pour repérer ces personnes, la loi permettra d’étendre la surveillance électronique à toutes les personnes en contact avec des personnes déjà suspectées. En analysant leurs contacts, la fréquence de ces derniers et les modes de communication, les services de renseignement espèrent pouvoir détecter ces nouveaux profils en amont.

        De nouveaux outils et méthodes de collecte

        Les services pourront également procéder, après un avis de la CNCTR, à la pose de micros dans une pièce ou de mouchards sur un objet (voiture par exemple), ou à l’intérieur d’un ordinateur. L’utilisation des IMSI-catchers (fausses antennes qui permettent d’intercepter des conversations téléphoniques) est également légalisée, pour les services de renseignement, dans certains cas. Le nombre maximal de ces appareils sera fixé par arrêté du premier ministre après l’avis de la CNCTR.

        Lire : Que sont les IMSI-catchers, ces valises qui espionnent les téléphones portables ?

        La loi introduit également des mesures de surveillance internationale : concrètement, les procédures de contrôle seront allégées lorsqu’un des « bouts » de la communication sera situé à l’étranger (concrètement, un Français qui parle avec un individu situé à l’étranger). Cependant, comme l’a souligné l’Arcep (l’Autorité de régulation des communications électroniques et des postes), sollicitée pour le versant technique de cette mesure, il est parfois difficile de s’assurer qu’une communication, même passant par l’étranger, ne concerne pas deux Français.

        Un nouveau fichier

        La loi crée un fichier judiciaire national automatisé des auteurs d’infractions terroristes (Fijait), dont les données pourront être conservées pendant vingt ans.

        Ce fichier concerne les personnes ayant été condamnées, même si une procédure d’appel est en cours. Les mineurs pourront aussi être inscrits dans ce fichier et leurs données conservées jusqu’à dix ans. L’inscription ne sera pas automatique et se fera sur décision judiciaire. Certaines mises en examen pourront aussi apparaître sur ce fichier. En cas de non-lieu, relaxe, acquittement, amnistie ou réhabilitation, ces informations seront effacées.

        Renseignement pénitentiaire

        Le renseignement pénitentiaire pourra, dans des conditions qui seront fixées par décret, profiter des techniques que légalise le projet de loi pour les services de renseignement. La ministre de la justice, Christiane Taubira, était défavorable à cette disposition, soutenue par le rapporteur du texte, la droite et une partie des députés de gauche. Pour la ministre, cette innovation va dénaturer le renseignement pénitentiaire et le transformer en véritable service de renseignement.

        Conservation des données

        La CNIL a fait part à plusieurs reprises de sa volonté d’exercer sa mission de contrôle sur les fichiers liés au renseignement, qui seront alimentés par ces collectes. Ces fichiers sont aujourd’hui exclus du périmètre d’action de la CNIL.

        La durée de conservation des données collectées – et l’adaptation de cette durée à la technique employée – a par ailleurs été inscrite dans la loi, contrairement au projet initial du gouvernement qui entendait fixer ces limites par décret. Elle pourra aller jusqu’à cinq ans dans le cas des données de connexion.

        Un dispositif pour les lanceurs d’alerte

        La loi prévoit également une forme de protection pour les agents qui seraient témoins de surveillance illégale. Ces lanceurs d’alerte pourraient solliciter la CNCTR, voire le premier ministre, et leur fournir toutes les pièces utiles. La CNCTR pourra ensuite aviser le procureur de la République et solliciter la Commission consultative du secret de la défense nationale afin que cette dernière « donne au premier ministre son avis sur la possibilité de déclassifier tout ou partie de ces éléments ». Aucune mesure de rétorsion ne pourra viser l’agent qui aurait dénoncé des actes potentiellement illégaux.

        Piratage de Sony Pictures ▪ Fibre optique ▪ Gouvernance du Net ▪ The surveillance files ▪ Neutralité du Net ▪ SXSW ▪ Hadopi ▪ Sécurité informatique ▪ Cyberespionnage ▪ Drones
         
        En Continu
        ================================================ FILE: test/test-pages/liberation-1/expected-metadata.json ================================================ { "title": "Un troisième Français mort dans le séisme au Népal", "byline": "Par Sébastien Farcis", "dir": null, "lang": "fr", "excerpt": "Laurent Fabius a accueilli jeudi matin à Roissy un premier avion spécial ramenant des rescapés.", "siteName": "Libération.fr", "publishedTime": "2015-04-30T07:19:58", "readerable": true } ================================================ FILE: test/test-pages/liberation-1/expected.html ================================================

        Un troisième Français a été tué dans le tremblement de terre samedi au Népal, emporté par une avalanche, a déclaré jeudi le ministre des Affaires étrangères. Les autorités françaises sont toujours sans nouvelles «d’encore plus de 200» personnes. «Pour certains d’entre eux on est très interrogatif», a ajouté Laurent Fabius. Il accueillait à Roissy un premier avion spécial ramenant des rescapés. L’Airbus A350 affrété par les autorités françaises s’est posé peu avant 5h45 avec à son bord 206 passagers, dont 12 enfants et 26 blessés, selon une source du Quai d’Orsay. Quasiment tous sont français, à l’exception d’une quinzaine de ressortissants allemands, suisses, italiens, portugais ou encore turcs. Des psychologues, une équipe médicale et des personnels du centre de crise du Quai d’Orsay les attendent.

        L’appareil, mis à disposition par Airbus, était arrivé à Katmandou mercredi matin avec 55 personnels de santé et humanitaires, ainsi que 25 tonnes de matériel (abris, médicaments, aide alimentaire). Un deuxième avion dépêché par Paris, qui était immobilisé aux Emirats depuis mardi avec 20 tonnes de matériel, est arrivé jeudi à Katmandou, dont le petit aéroport est engorgé par le trafic et l’afflux d’aide humanitaire. Il devait lui aussi ramener des Français, «les plus éprouvés» par la catastrophe et les «plus vulnérables (blessés, familles avec enfants)», selon le ministère des Affaires étrangères.

        2 209 Français ont été localisés sains et saufs tandis que 393 n’ont pas encore pu être joints, selon le Quai d’Orsay. Environ 400 Français ont demandé à être rapatriés dans les vols mis en place par la France.

        Le séisme a fait près de 5 500 morts et touche huit des 28 millions d’habitants du Népal. Des dizaines de milliers de personnes sont sans abri.


        ================================================ FILE: test/test-pages/liberation-1/source.html ================================================ Un troisième Français mort dans le séisme au Népal - Libération

        Un troisième Français a été tué dans le tremblement de terre samedi au Népal, emporté par une avalanche, a déclaré jeudi le ministre des Affaires étrangères. Les autorités françaises sont toujours sans nouvelles «d’encore plus de 200» personnes. «Pour certains d’entre eux on est très interrogatif», a ajouté Laurent Fabius. Il accueillait à Roissy un premier avion spécial ramenant des rescapés. L’Airbus A350 affrété par les autorités françaises s’est posé peu avant 5h45 avec à son bord 206 passagers, dont 12 enfants et 26 blessés, selon une source du Quai d’Orsay. Quasiment tous sont français, à l’exception d’une quinzaine de ressortissants allemands, suisses, italiens, portugais ou encore turcs. Des psychologues, une équipe médicale et des personnels du centre de crise du Quai d’Orsay les attendent.

        L’appareil, mis à disposition par Airbus, était arrivé à Katmandou mercredi matin avec 55 personnels de santé et humanitaires, ainsi que 25 tonnes de matériel (abris, médicaments, aide alimentaire). Un deuxième avion dépêché par Paris, qui était immobilisé aux Emirats depuis mardi avec 20 tonnes de matériel, est arrivé jeudi à Katmandou, dont le petit aéroport est engorgé par le trafic et l’afflux d’aide humanitaire. Il devait lui aussi ramener des Français, «les plus éprouvés» par la catastrophe et les «plus vulnérables (blessés, familles avec enfants)», selon le ministère des Affaires étrangères.

        2 209 Français ont été localisés sains et saufs tandis que 393 n’ont pas encore pu être joints, selon le Quai d’Orsay. Environ 400 Français ont demandé à être rapatriés dans les vols mis en place par la France.

        Le séisme a fait près de 5 500 morts et touche huit des 28 millions d’habitants du Népal. Des dizaines de milliers de personnes sont sans abri.


        AFP
        ================================================ FILE: test/test-pages/lifehacker-post-comment-load/expected-metadata.json ================================================ { "title": "How to Program Your Mind to Stop Buying Crap You Don’t Need", "byline": "Patrick Allan", "dir": null, "lang": "en-US", "excerpt": "We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.", "siteName": "Lifehacker", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/lifehacker-post-comment-load/expected.html ================================================

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.

        Understand How Your Own Brain Works Against You

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        It may come as no surprise to learn that stores employ all kinds of tricks to get you to part ways with your cash, and your brain plays right along. Through psychological tricks, product placement, and even color, stores are designed from the ground up to increase spending. We've talked about the biggest things stores do to manipulate your senses, but here are some of the biggest things to look out for:

        • Color: Stores use color to make products attractive and eye-catching, but they also use color on price labels. Red stands out and can encourage taking action, that's why it's commonly associated with sale signage and advertising. When you see red, remember what they're trying to do to your brain with that color. You don't to buy something just because it's on sale.
        • Navigation Roadblocks: Stores force you to walk around stuff you don't need to find the stuff you are really after. Have a list of what you need before you go in, go straight to it, and imagine it's the only item in the store.
        • The Touch Factor: Stores place items they want to sell in easy to reach locations and encourage you to touch them. Don't do it! As soon as you pick something up, you're more likely to buy it because your mind suddenly takes ownership of the object. Don't pick anything up and don't play with display items.
        • Scents and Sounds: You'll probably hear classic, upbeat tunes when you walk into a store. The upbeat music makes you happy and excited, while playing familiar songs makes you feel comfortable. They also use pleasant smells to put your mind at ease. A happy, comfortable mind at ease is a dangerous combination for your brain when shopping. There's not much you can do to avoid this unless you shop online, but it's good to be aware of it.

        And sure, we can blame the stores all we want, but you won't change how they operate—you can only be aware of how your brain is falling for their tricks. Even without the stores, your brain is working against you on its own, thanks to some simple cognitive biases.

        For example, confirmation bias makes you only believe the information that conforms to your prior beliefs, while you discount everything else. Advertisers appeal to this bias directly by convincing you one item is better than another with imagery and other tricks, regardless of what hard facts might say. Keep your mind open, do your own research, and accept when you're wrong about a product. The Decoy effect is also a commonly used tactic. You think one product is a deal because it's next to a similar product that's priced way higher. Even if it's a product you need, it's probably not as good of a deal as it looks right then and there. Again, always research beforehand and be on the lookout for this common trick to avoid impulse buys.

        Make a List of Everything You Own and Do Some Decluttering

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now that you know what you're up against, it's time to start changing the way you think. Before you can stop buying crap you don't need, you need to identify what that crap is. The first step is to make a list of every single thing you own. Every. Single. Thing. This might sound extreme, but you need to gather your data so you can start reprogramming your mind.

        The purpose of this exercise is twofold: you see what you already have and don't need to ever buy again, and you get to see what you shouldn't have bought in the first place. As you list everything out, separate items into categories. It's extremely important that you are as honest with yourself as possible while you do this. It's also important you actually write this all down or type it all out. Here is the first set of categories to separate everything into:

        • Need: You absolutely need this item to get by on a day to day basis.
        • Sometimes Need: You don't need this item every day, but you use it on a somewhat regular basis.
        • Want: You bought this item because you wanted it, not because you needed it.
        • Crap: You don't have a good reason why you have it and you already know it needs to go (there's probably a few of these items, at least).

        Leave the things you listed as "needs" alone, put your stuff listed as "crap" in a pile or box to go bye-bye, and move your attention back to your "sometimes need" and "want" lists. You need to go back over both of those lists because you probably fudged some of the listings, either subconsciously or intentionally. Now ask yourself these three questions as you go through both the "sometimes need" and "want" lists:

        • When was the last time I used this?
        • When will I use this again?
        • Does this item bring you joy?

        Remember to be honest and adjust your lists accordingly. There's nothing wrong with keeping things you wanted. Material items can bring happiness to many people, but make sure the items on your "want" list actively provide you joy and are being used. If an item doesn't get much use or doesn't make you happy, add it to the "crap" list.

        Once you have everything organized, it's time to do some serious decluttering. This listing exercise should get you started, but there are a lot of other great ideas when it comes to ditching the junk you don't need. Regardless, everything on your "crap" list needs to go. You can donate it, sell it at a yard sale, give it away to people know, whatever you like. Before you get rid of everything, though, take a picture of all your stuff together. Print out or save the picture somewhere. Some of it was probably gifts, but in general, this is all the crap you bought that you don't need. Take a good look and remember it.

        See How Much Money and Time You Spent on the Stuff You Threw Out

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now take a look at your "crap" list again and start calculating how much you spent on all of it. If it was a gift, mark it as $0. Otherwise, figure out the price of the item at the time you bought it. If you got a deal or bought it on sale it's okay to factor that in, but try to be as accurate as possible. Once you have the price for each item, add it all together. Depending on your spending habits this could possibly be in the hundreds to thousands of dollars. Remember the picture you took of all this stuff? Attach the total cost to the picture so you can see both at the same time.

        With the money cost figured out, you should take a look at the other costs too. Time is a resource just like any other, and it's a finite one. What kind of time did you pour into these things? Consider the time you spent acquiring and using these items, then write it all down. These can be rough estimations, but go ahead and add it all up when you think you've got it. Now attach the total time to same picture as before and think of the other ways you could have spent all that time. This isn't to make you feel bad about yourself, just to deliver information to your brain in an easy-to-understand form. When you look at it all like this, it can open your eyes a little more, and help you think about purchases in the future. You'll look at an item and ask yourself, "Will this just end up in the picture?"

        List Every Non-Material Thing In Your Life that Makes You Happy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now it's time to make a different list. While material items may bring plenty of joy, the things in your life that make you happiest probably can't be bought. Get a separate piece of paper or create a new document and list out everything in your life that makes you happy. If you can't buy it, it's eligible for the list. It doesn't matter if it only makes you crack a smile or makes you jump for joy, list it out.

        These are probably the things that actually make you want to get out of bed in the morning and keep on keepin' on. Once you have it all down, put it in your purse or wallet. The next time you feel the urge to buy something, whip this list out first and remind yourself why you probably don't need it.

        Spend Some Time Away from Material Things to Gain Perspective

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you're having a really hard time with your spending, it can help to get away from material objects completely. When you're constantly surrounded by stuff and have access to buying things at all times, it can be really tough to break the habit. Spend a day in the park enjoying the sights and sounds of the outdoors, go camping with some friends, or hike a trail you haven't been on before.

        Essentially, you want to show yourself that you don't need your "things" to have a good time. When you realize how much fun you can have without all the trinkets and trivets, you'll start to shut down your desire to buy them. If you can't get really get away right now, just go for a walk without your purse or wallet (but carry your ID). If you can't buy anything, you'll be forced to experience things a different way.

        Develop a Personal "Should I Buy This?" Test

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you don't have a personal "should I buy this?" test, now's the perfect time to make one. When you find an item you think you need or want, it has to pass all of the questions you have on your test before you can buy it. Here's where you can use all of the data you've gathered so far and put it to really good use. The test should be personalized to your own buying habits, but here are some example questions:

        • Is this a planned purchase?
        • Will it end up in the "crap" list picture one day?
        • Where am I going to put it?
        • Have I included this in my budget?
        • Why do I want/need it?

        Custom build your test to hit all of your weaknesses. If you make a lot of impulse buys, include questions that address that. If you experience a lot of buyer's remorse, include a lot of questions that make you think about the use of item after you buy it. If buying the latest and greatest technology is your weakness, Joshua Becker at Becoming Minimalist suggests you ask yourself what problem the piece of tech solves. If you can't think of anything it solves or if you already have something that solves it, you don't need it. Be thorough and build a test that you can run through your mind every time you consider buying something.

        Learn to Delay Gratification and Destroy the Urge to Impulse Buy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        When it comes to the unnecessary crap we buy, impulse purchases probably make up a good deal of them. We love to feel gratification instantly and impulse buys appeal to that with a rush of excitement with each new purchase. We like to believe that we have control over our impulses all the time, but we really don't, and that's a major problem for the ol' wallet.

        The key is teaching your brain that it's okay to wait for gratification. You can do this with a simple time out every time you want something. Look at whatever you're thinking of buying, go through your personal "should I buy this?" test, and then walk away for a little while. Planning your purchases ahead is ideal, so the longer you can hold off, the better. Set yourself a reminder to check on the item a week or month down the line. When you come back to it, you may find that you don't even want it, just the gratification that would come with it. If you're shopping online, you can do the same thing. Walk away from your desk or put your phone in your pocket and do something else for a little while.

        You can also avoid online impulse purchases by making it harder to do. Block shopping web sites during time periods you know you're at your weakest, or remove all of your saved credit card or Paypal information. You can also practice the "HALT" method when you're shopping online or in a store. Try not to buy things when you're Hungry, Angry, Lonely, or Tired because you're at your weakest state mentally. Last, but not least, the "stranger test" can help you weed out bad purchases too.

        The last thing you should consider when it comes to impulse buys is "artificial replacement." As Trent Hamm at The Simple Dollar explains, artificial replacement can happen when you start to reduce the time you get with your main interests:

        Whenever I consistently cut quality time for my main interests out of my life, I start to long for them. As you saw in that "typical" day, I do make room for spending time with my family, but my other two main interests are absent. If that happens too many days in a row, I start to really miss reading. I start to really miss playing thoughtful board games with friends. What happens after that? I start to substitute. When I don't have the opportunity to sit down for an hour or even for half an hour and really get lost in a book, I start looking for an alternative way to fill in the tiny slices of time that I do have. I'll spend money.

        You probably have things in your life that provide plenty of gratification, so don't get caught substituting it with impulse buys. Always make sure you keep yourself happy with plenty of time doing the things you like to do and you won't be subconsciously trying to fill that void with useless crap.

        Turn the Money You Save Into More Money

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Once you've programmed your mind to stop buying crap you don't need, you'll have some extra cash to play with. Take all that money and start putting it toward your future and things you will need further down the road. You might need a home, a vehicle, or a way to retire, but none of that can happen until you start planning for it.

        Start by paying off any debts you already have. Credit cards, student loans, and even car payments can force you to live paycheck to paycheck. Use the snowball method and pay off some small balances to make you feel motivated, then start taking out your debt in full force with the stacking method: stop creating new debt, determine which balances have the highest interest rates, and create a payment schedule to pay them off efficiently.

        With your debts whittled down, you should start an emergency fund. No matter how well you plan things, accidents and health emergencies can still happen. An emergency fund is designed to make those kinds of events more manageable. This type of savings account is strictly for when life throws you a curveball, but you can grow one pretty easily with only modest savings.

        When you've paid off your debt and prepared yourself for troubled times, you can start saving for the big stuff. All that money you're not spending on crap anymore can be saved, invested, and compounded to let you buy comfort and security. If you don't know where to start, talk to a financial planner. Or create a simple, yet effective "set and forget" investment portfolio. You've worked hard to reprogram your mind, so make sure you reap the benefits for many years to come.

        Photos by cmgirl (Shutterstock), Macrovector (Shutterstock), J E Theriot, davidd, George Redgrave, David Amsler, Arup Malakar, J B, jakerome, 401(K) 2012.

        ================================================ FILE: test/test-pages/lifehacker-post-comment-load/source.html ================================================ How to Program Your Mind to Stop Buying Crap You Don’t Need
        Trending on Kinja
        1. 1.
          Aaron Hernandez Is An Idiot, But He Still Might Get Away With Murder
          Aaron Hernandez Is An Idiot, But He Still Might Get Away With Murder
          on Deadspin
        2. 2.
          Top Gear Hosts James May And Richard Hammond React To Clarkson Firing
          on Jalopnik
        3. 3.
          Jeremy Clarkson Officially Fired From Top Gear
          on Jalopnik

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        321,046
        36
        PatrickAllan

        Patrick Allan

        ProfileFollowUnfollow
        Patrick Allan
        Filed to: money
        3/09/15 8:00am 3/09/15 8:00am

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.

        Understand How Your Own Brain Works Against You

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        It may come as no surprise to learn that stores employ all kinds of tricks to get you to part ways with your cash, and your brain plays right along. Through psychological tricks, product placement, and even color, stores are designed from the ground up to increase spending. We've talked about the biggest things stores do to manipulate your senses, but here are some of the biggest things to look out for:

        • Color: Stores use color to make products attractive and eye-catching, but they also use color on price labels. Red stands out and can encourage taking action, that's why it's commonly associated with sale signage and advertising. When you see red, remember what they're trying to do to your brain with that color. You don't to buy something just because it's on sale.
        • Navigation Roadblocks: Stores force you to walk around stuff you don't need to find the stuff you are really after. Have a list of what you need before you go in, go straight to it, and imagine it's the only item in the store.
        • The Touch Factor: Stores place items they want to sell in easy to reach locations and encourage you to touch them. Don't do it! As soon as you pick something up, you're more likely to buy it because your mind suddenly takes ownership of the object. Don't pick anything up and don't play with display items.
        • Scents and Sounds: You'll probably hear classic, upbeat tunes when you walk into a store. The upbeat music makes you happy and excited, while playing familiar songs makes you feel comfortable. They also use pleasant smells to put your mind at ease. A happy, comfortable mind at ease is a dangerous combination for your brain when shopping. There's not much you can do to avoid this unless you shop online, but it's good to be aware of it.

        And sure, we can blame the stores all we want, but you won't change how they operate—you can only be aware of how your brain is falling for their tricks. Even without the stores, your brain is working against you on its own, thanks to some simple cognitive biases.

        For example, confirmation bias makes you only believe the information that conforms to your prior beliefs, while you discount everything else. Advertisers appeal to this bias directly by convincing you one item is better than another with imagery and other tricks, regardless of what hard facts might say. Keep your mind open, do your own research, and accept when you're wrong about a product. The Decoy effect is also a commonly used tactic. You think one product is a deal because it's next to a similar product that's priced way higher. Even if it's a product you need, it's probably not as good of a deal as it looks right then and there. Again, always research beforehand and be on the lookout for this common trick to avoid impulse buys.

        Make a List of Everything You Own and Do Some Decluttering

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now that you know what you're up against, it's time to start changing the way you think. Before you can stop buying crap you don't need, you need to identify what that crap is. The first step is to make a list of every single thing you own. Every. Single. Thing. This might sound extreme, but you need to gather your data so you can start reprogramming your mind.

        The purpose of this exercise is twofold: you see what you already have and don't need to ever buy again, and you get to see what you shouldn't have bought in the first place. As you list everything out, separate items into categories. It's extremely important that you are as honest with yourself as possible while you do this. It's also important you actually write this all down or type it all out. Here is the first set of categories to separate everything into:

        • Need: You absolutely need this item to get by on a day to day basis.
        • Sometimes Need: You don't need this item every day, but you use it on a somewhat regular basis.
        • Want: You bought this item because you wanted it, not because you needed it.
        • Crap: You don't have a good reason why you have it and you already know it needs to go (there's probably a few of these items, at least).

        Leave the things you listed as "needs" alone, put your stuff listed as "crap" in a pile or box to go bye-bye, and move your attention back to your "sometimes need" and "want" lists. You need to go back over both of those lists because you probably fudged some of the listings, either subconsciously or intentionally. Now ask yourself these three questions as you go through both the "sometimes need" and "want" lists:

        • When was the last time I used this?
        • When will I use this again?
        • Does this item bring you joy?

        Remember to be honest and adjust your lists accordingly. There's nothing wrong with keeping things you wanted. Material items can bring happiness to many people, but make sure the items on your "want" list actively provide you joy and are being used. If an item doesn't get much use or doesn't make you happy, add it to the "crap" list.

        Once you have everything organized, it's time to do some serious decluttering. This listing exercise should get you started, but there are a lot of other great ideas when it comes to ditching the junk you don't need. Regardless, everything on your "crap" list needs to go. You can donate it, sell it at a yard sale, give it away to people know, whatever you like. Before you get rid of everything, though, take a picture of all your stuff together. Print out or save the picture somewhere. Some of it was probably gifts, but in general, this is all the crap you bought that you don't need. Take a good look and remember it.

        See How Much Money and Time You Spent on the Stuff You Threw Out

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now take a look at your "crap" list again and start calculating how much you spent on all of it. If it was a gift, mark it as $0. Otherwise, figure out the price of the item at the time you bought it. If you got a deal or bought it on sale it's okay to factor that in, but try to be as accurate as possible. Once you have the price for each item, add it all together. Depending on your spending habits this could possibly be in the hundreds to thousands of dollars. Remember the picture you took of all this stuff? Attach the total cost to the picture so you can see both at the same time.

        With the money cost figured out, you should take a look at the other costs too. Time is a resource just like any other, and it's a finite one. What kind of time did you pour into these things? Consider the time you spent acquiring and using these items, then write it all down. These can be rough estimations, but go ahead and add it all up when you think you've got it. Now attach the total time to same picture as before and think of the other ways you could have spent all that time. This isn't to make you feel bad about yourself, just to deliver information to your brain in an easy-to-understand form. When you look at it all like this, it can open your eyes a little more, and help you think about purchases in the future. You'll look at an item and ask yourself, "Will this just end up in the picture?"

        List Every Non-Material Thing In Your Life that Makes You Happy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now it's time to make a different list. While material items may bring plenty of joy, the things in your life that make you happiest probably can't be bought. Get a separate piece of paper or create a new document and list out everything in your life that makes you happy. If you can't buy it, it's eligible for the list. It doesn't matter if it only makes you crack a smile or makes you jump for joy, list it out.

        These are probably the things that actually make you want to get out of bed in the morning and keep on keepin' on. Once you have it all down, put it in your purse or wallet. The next time you feel the urge to buy something, whip this list out first and remind yourself why you probably don't need it.

        Spend Some Time Away from Material Things to Gain Perspective

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you're having a really hard time with your spending, it can help to get away from material objects completely. When you're constantly surrounded by stuff and have access to buying things at all times, it can be really tough to break the habit. Spend a day in the park enjoying the sights and sounds of the outdoors, go camping with some friends, or hike a trail you haven't been on before.

        Essentially, you want to show yourself that you don't need your "things" to have a good time. When you realize how much fun you can have without all the trinkets and trivets, you'll start to shut down your desire to buy them. If you can't get really get away right now, just go for a walk without your purse or wallet (but carry your ID). If you can't buy anything, you'll be forced to experience things a different way.

        Develop a Personal "Should I Buy This?" Test

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you don't have a personal "should I buy this?" test, now's the perfect time to make one. When you find an item you think you need or want, it has to pass all of the questions you have on your test before you can buy it. Here's where you can use all of the data you've gathered so far and put it to really good use. The test should be personalized to your own buying habits, but here are some example questions:

        • Is this a planned purchase?
        • Will it end up in the "crap" list picture one day?
        • Where am I going to put it?
        • Have I included this in my budget?
        • Why do I want/need it?

        Custom build your test to hit all of your weaknesses. If you make a lot of impulse buys, include questions that address that. If you experience a lot of buyer's remorse, include a lot of questions that make you think about the use of item after you buy it. If buying the latest and greatest technology is your weakness, Joshua Becker at Becoming Minimalist suggests you ask yourself what problem the piece of tech solves. If you can't think of anything it solves or if you already have something that solves it, you don't need it. Be thorough and build a test that you can run through your mind every time you consider buying something.

        Learn to Delay Gratification and Destroy the Urge to Impulse Buy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        When it comes to the unnecessary crap we buy, impulse purchases probably make up a good deal of them. We love to feel gratification instantly and impulse buys appeal to that with a rush of excitement with each new purchase. We like to believe that we have control over our impulses all the time, but we really don't, and that's a major problem for the ol' wallet.

        The key is teaching your brain that it's okay to wait for gratification. You can do this with a simple time out every time you want something. Look at whatever you're thinking of buying, go through your personal "should I buy this?" test, and then walk away for a little while. Planning your purchases ahead is ideal, so the longer you can hold off, the better. Set yourself a reminder to check on the item a week or month down the line. When you come back to it, you may find that you don't even want it, just the gratification that would come with it. If you're shopping online, you can do the same thing. Walk away from your desk or put your phone in your pocket and do something else for a little while.

        You can also avoid online impulse purchases by making it harder to do. Block shopping web sites during time periods you know you're at your weakest, or remove all of your saved credit card or Paypal information. You can also practice the "HALT" method when you're shopping online or in a store. Try not to buy things when you're Hungry, Angry, Lonely, or Tired because you're at your weakest state mentally. Last, but not least, the "stranger test" can help you weed out bad purchases too.

        The last thing you should consider when it comes to impulse buys is "artificial replacement." As Trent Hamm at The Simple Dollar explains, artificial replacement can happen when you start to reduce the time you get with your main interests:

        Whenever I consistently cut quality time for my main interests out of my life, I start to long for them. As you saw in that "typical" day, I do make room for spending time with my family, but my other two main interests are absent. If that happens too many days in a row, I start to really miss reading. I start to really miss playing thoughtful board games with friends. What happens after that? I start to substitute. When I don't have the opportunity to sit down for an hour or even for half an hour and really get lost in a book, I start looking for an alternative way to fill in the tiny slices of time that I do have. I'll spend money.

        You probably have things in your life that provide plenty of gratification, so don't get caught substituting it with impulse buys. Always make sure you keep yourself happy with plenty of time doing the things you like to do and you won't be subconsciously trying to fill that void with useless crap.

        Turn the Money You Save Into More Money

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Once you've programmed your mind to stop buying crap you don't need, you'll have some extra cash to play with. Take all that money and start putting it toward your future and things you will need further down the road. You might need a home, a vehicle, or a way to retire, but none of that can happen until you start planning for it.

        Start by paying off any debts you already have. Credit cards, student loans, and even car payments can force you to live paycheck to paycheck. Use the snowball method and pay off some small balances to make you feel motivated, then start taking out your debt in full force with the stacking method: stop creating new debt, determine which balances have the highest interest rates, and create a payment schedule to pay them off efficiently.

        With your debts whittled down, you should start an emergency fund. No matter how well you plan things, accidents and health emergencies can still happen. An emergency fund is designed to make those kinds of events more manageable. This type of savings account is strictly for when life throws you a curveball, but you can grow one pretty easily with only modest savings.

        When you've paid off your debt and prepared yourself for troubled times, you can start saving for the big stuff. All that money you're not spending on crap anymore can be saved, invested, and compounded to let you buy comfort and security. If you don't know where to start, talk to a financial planner. Or create a simple, yet effective "set and forget" investment portfolio. You've worked hard to reprogram your mind, so make sure you reap the benefits for many years to come.

        Photos by cmgirl (Shutterstock), Macrovector (Shutterstock), J E Theriot, davidd, George Redgrave, David Amsler, Arup Malakar, J B, jakerome, 401(K) 2012.

        36 146Reply
        Patrick Allan’s Discussions
        All replies
        jnovo22

        jnovo22

        ProfileFollowUnfollow
        jnovo22Patrick Allan
        3/09/15 8:12am 3/09/15 8:12am

        This is one of my hardest habits to break. I've been working on it for literally years. I'm getting better, mainly because I've gone to cash-only recently. Swiping the debit/credit cards don't seem like "real" money (even though it obviously is). Having cash on hand, and then see it disappear makes the "Should I buy This?" decision that much easier. Usually it's "No".

        924Reply
        Flagged
        jnovo22

        jnovo22

        ProfileFollowUnfollow
        jnovo22Patrick Allan
        3/09/15 8:47am 3/09/15 8:47am

        Yup. Each week, I give myself $25 cash for spending on me. If I go out to eat Friday night, that's it, it's gone.

        I use the envelope system for dividing saved cash for special purchases, home improvements, etc. It's crazy how fast it adds up.

        12Reply
        Flagged
        thegrayadder

        The Gray Adder

        ProfileFollowUnfollow
        The Gray AdderPatrick Allan
        3/09/15 8:26am 3/09/15 8:26am

        Make a List of Everything You Own and Do Some Decluttering

        Why would I make a list when I could get right down to the business of decluttering?

        31Reply
        Flagged
        PatrickAllan

        Patrick Allan

        ProfileFollowUnfollow
        Patrick AllanThe Gray Adder
        3/09/15 8:32am 3/09/15 8:32am

        So you know what you already have. You can avoid buying something that you forget you had, or the next time you're in a store, you might remember that you have an item that can do the same job as the thing you're interested in. It also helps you gain a little perspective when you realize how many things you actually own already. It can help you rethink those desires to buy things.

        111Reply
        Flagged
        winfredarmandelafuentelati

        greenarcher02

        ProfileFollowUnfollow
        greenarcher02Patrick Allan
        3/09/15 8:46am 3/09/15 8:46am

        I am. Well, I was..... Some of it was spent on impulse buying Ziiiro watches last Valentine's Day lol. :( So I'm saving up, again. Sucks that it'll get delayed because of a freaking watch.

        Ugh I feel like I need to read this article everyday.

        6Reply
        Flagged
        timgray

        timgray

        ProfileFollowUnfollow
        timgrayPatrick Allan
        3/09/15 9:00am 3/09/15 9:00am

        I love the suggestion of going camping to get away from stuff... You do realize that camping gear can be more expensive than electronics and gadgets.... Now I need to buy a $4500 tent and $1200 hiking shoes to get away from things....

        Oh and I have to have all matching camping clothing from North Face... I need to go shopping.

        427Reply
        Flagged
        PatrickAllan

        Patrick Allan

        ProfileFollowUnfollow
        Patrick Allantimgray
        3/09/15 9:07am 3/09/15 9:07am

        Haha, yeah, some camping gear can be pretty pricey, but I just went this past weekend with a rented tent and some friends, and it didn't cost much. Just got to avoid the "I want to look like an outdoorsy camper" style I guess :P

        18Reply
        Flagged
        timgray

        timgray

        ProfileFollowUnfollow
        timgrayPatrick Allan
        3/09/15 9:25am 3/09/15 9:25am

        My camping gear addiction is one of those that I need to do some decluttering on. I have a really cool 2 person Tent Cot that sound like awesome ideas until you actually see one in person.

        http://www.amazon.com/Kamp-Rite-TB34...

        They weigh nearly 90 pounds and will not fit in the back of anything but a full size SUV with the seats folded down or a pickup truck.

        So it has sat in it's box in my basement for the past 3 years as a testament to my UPS guys hernia when he delivered it.

        5Reply
        Flagged
        PatrickAllan

        Patrick Allan

        ProfileFollowUnfollow
        Patrick Allantimgray
        3/09/15 9:37am 3/09/15 9:37am

        lol! That does sound like a cool idea, but you're right, what's the point if you can't transport it easily? I wonder if someone makes a lighter, less cumbersome model...

        1Reply
        Flagged
        Hosts of other popular chats
        View Patrick Allan's discussionsView all 146 replies
        Powered by

        Kinja

        ================================================ FILE: test/test-pages/lifehacker-working/expected-metadata.json ================================================ { "title": "How to Program Your Mind to Stop Buying Crap You Don’t Need", "byline": "Patrick Allan", "dir": null, "lang": "en-US", "excerpt": "We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.", "siteName": "Lifehacker", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/lifehacker-working/expected.html ================================================

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.

        Understand How Your Own Brain Works Against You

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        It may come as no surprise to learn that stores employ all kinds of tricks to get you to part ways with your cash, and your brain plays right along. Through psychological tricks, product placement, and even color, stores are designed from the ground up to increase spending. We've talked about the biggest things stores do to manipulate your senses, but here are some of the biggest things to look out for:

        • Color: Stores use color to make products attractive and eye-catching, but they also use color on price labels. Red stands out and can encourage taking action, that's why it's commonly associated with sale signage and advertising. When you see red, remember what they're trying to do to your brain with that color. You don't to buy something just because it's on sale.
        • Navigation Roadblocks: Stores force you to walk around stuff you don't need to find the stuff you are really after. Have a list of what you need before you go in, go straight to it, and imagine it's the only item in the store.
        • The Touch Factor: Stores place items they want to sell in easy to reach locations and encourage you to touch them. Don't do it! As soon as you pick something up, you're more likely to buy it because your mind suddenly takes ownership of the object. Don't pick anything up and don't play with display items.
        • Scents and Sounds: You'll probably hear classic, upbeat tunes when you walk into a store. The upbeat music makes you happy and excited, while playing familiar songs makes you feel comfortable. They also use pleasant smells to put your mind at ease. A happy, comfortable mind at ease is a dangerous combination for your brain when shopping. There's not much you can do to avoid this unless you shop online, but it's good to be aware of it.

        And sure, we can blame the stores all we want, but you won't change how they operate—you can only be aware of how your brain is falling for their tricks. Even without the stores, your brain is working against you on its own, thanks to some simple cognitive biases.

        For example, confirmation bias makes you only believe the information that conforms to your prior beliefs, while you discount everything else. Advertisers appeal to this bias directly by convincing you one item is better than another with imagery and other tricks, regardless of what hard facts might say. Keep your mind open, do your own research, and accept when you're wrong about a product. The Decoy effect is also a commonly used tactic. You think one product is a deal because it's next to a similar product that's priced way higher. Even if it's a product you need, it's probably not as good of a deal as it looks right then and there. Again, always research beforehand and be on the lookout for this common trick to avoid impulse buys.

        Make a List of Everything You Own and Do Some Decluttering

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now that you know what you're up against, it's time to start changing the way you think. Before you can stop buying crap you don't need, you need to identify what that crap is. The first step is to make a list of every single thing you own. Every. Single. Thing. This might sound extreme, but you need to gather your data so you can start reprogramming your mind.

        The purpose of this exercise is twofold: you see what you already have and don't need to ever buy again, and you get to see what you shouldn't have bought in the first place. As you list everything out, separate items into categories. It's extremely important that you are as honest with yourself as possible while you do this. It's also important you actually write this all down or type it all out. Here is the first set of categories to separate everything into:

        • Need: You absolutely need this item to get by on a day to day basis.
        • Sometimes Need: You don't need this item every day, but you use it on a somewhat regular basis.
        • Want: You bought this item because you wanted it, not because you needed it.
        • Crap: You don't have a good reason why you have it and you already know it needs to go (there's probably a few of these items, at least).

        Leave the things you listed as "needs" alone, put your stuff listed as "crap" in a pile or box to go bye-bye, and move your attention back to your "sometimes need" and "want" lists. You need to go back over both of those lists because you probably fudged some of the listings, either subconsciously or intentionally. Now ask yourself these three questions as you go through both the "sometimes need" and "want" lists:

        • When was the last time I used this?
        • When will I use this again?
        • Does this item bring you joy?

        Remember to be honest and adjust your lists accordingly. There's nothing wrong with keeping things you wanted. Material items can bring happiness to many people, but make sure the items on your "want" list actively provide you joy and are being used. If an item doesn't get much use or doesn't make you happy, add it to the "crap" list.

        Once you have everything organized, it's time to do some serious decluttering. This listing exercise should get you started, but there are a lot of other great ideas when it comes to ditching the junk you don't need. Regardless, everything on your "crap" list needs to go. You can donate it, sell it at a yard sale, give it away to people know, whatever you like. Before you get rid of everything, though, take a picture of all your stuff together. Print out or save the picture somewhere. Some of it was probably gifts, but in general, this is all the crap you bought that you don't need. Take a good look and remember it.

        See How Much Money and Time You Spent on the Stuff You Threw Out

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now take a look at your "crap" list again and start calculating how much you spent on all of it. If it was a gift, mark it as $0. Otherwise, figure out the price of the item at the time you bought it. If you got a deal or bought it on sale it's okay to factor that in, but try to be as accurate as possible. Once you have the price for each item, add it all together. Depending on your spending habits this could possibly be in the hundreds to thousands of dollars. Remember the picture you took of all this stuff? Attach the total cost to the picture so you can see both at the same time.

        With the money cost figured out, you should take a look at the other costs too. Time is a resource just like any other, and it's a finite one. What kind of time did you pour into these things? Consider the time you spent acquiring and using these items, then write it all down. These can be rough estimations, but go ahead and add it all up when you think you've got it. Now attach the total time to same picture as before and think of the other ways you could have spent all that time. This isn't to make you feel bad about yourself, just to deliver information to your brain in an easy-to-understand form. When you look at it all like this, it can open your eyes a little more, and help you think about purchases in the future. You'll look at an item and ask yourself, "Will this just end up in the picture?"

        List Every Non-Material Thing In Your Life that Makes You Happy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now it's time to make a different list. While material items may bring plenty of joy, the things in your life that make you happiest probably can't be bought. Get a separate piece of paper or create a new document and list out everything in your life that makes you happy. If you can't buy it, it's eligible for the list. It doesn't matter if it only makes you crack a smile or makes you jump for joy, list it out.

        These are probably the things that actually make you want to get out of bed in the morning and keep on keepin' on. Once you have it all down, put it in your purse or wallet. The next time you feel the urge to buy something, whip this list out first and remind yourself why you probably don't need it.

        Spend Some Time Away from Material Things to Gain Perspective

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you're having a really hard time with your spending, it can help to get away from material objects completely. When you're constantly surrounded by stuff and have access to buying things at all times, it can be really tough to break the habit. Spend a day in the park enjoying the sights and sounds of the outdoors, go camping with some friends, or hike a trail you haven't been on before.

        Essentially, you want to show yourself that you don't need your "things" to have a good time. When you realize how much fun you can have without all the trinkets and trivets, you'll start to shut down your desire to buy them. If you can't get really get away right now, just go for a walk without your purse or wallet (but carry your ID). If you can't buy anything, you'll be forced to experience things a different way.

        Develop a Personal "Should I Buy This?" Test

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you don't have a personal "should I buy this?" test, now's the perfect time to make one. When you find an item you think you need or want, it has to pass all of the questions you have on your test before you can buy it. Here's where you can use all of the data you've gathered so far and put it to really good use. The test should be personalized to your own buying habits, but here are some example questions:

        • Is this a planned purchase?
        • Will it end up in the "crap" list picture one day?
        • Where am I going to put it?
        • Have I included this in my budget?
        • Why do I want/need it?

        Custom build your test to hit all of your weaknesses. If you make a lot of impulse buys, include questions that address that. If you experience a lot of buyer's remorse, include a lot of questions that make you think about the use of item after you buy it. If buying the latest and greatest technology is your weakness, Joshua Becker at Becoming Minimalist suggests you ask yourself what problem the piece of tech solves. If you can't think of anything it solves or if you already have something that solves it, you don't need it. Be thorough and build a test that you can run through your mind every time you consider buying something.

        Learn to Delay Gratification and Destroy the Urge to Impulse Buy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        When it comes to the unnecessary crap we buy, impulse purchases probably make up a good deal of them. We love to feel gratification instantly and impulse buys appeal to that with a rush of excitement with each new purchase. We like to believe that we have control over our impulses all the time, but we really don't, and that's a major problem for the ol' wallet.

        The key is teaching your brain that it's okay to wait for gratification. You can do this with a simple time out every time you want something. Look at whatever you're thinking of buying, go through your personal "should I buy this?" test, and then walk away for a little while. Planning your purchases ahead is ideal, so the longer you can hold off, the better. Set yourself a reminder to check on the item a week or month down the line. When you come back to it, you may find that you don't even want it, just the gratification that would come with it. If you're shopping online, you can do the same thing. Walk away from your desk or put your phone in your pocket and do something else for a little while.

        You can also avoid online impulse purchases by making it harder to do. Block shopping web sites during time periods you know you're at your weakest, or remove all of your saved credit card or Paypal information. You can also practice the "HALT" method when you're shopping online or in a store. Try not to buy things when you're Hungry, Angry, Lonely, or Tired because you're at your weakest state mentally. Last, but not least, the "stranger test" can help you weed out bad purchases too.

        The last thing you should consider when it comes to impulse buys is "artificial replacement." As Trent Hamm at The Simple Dollar explains, artificial replacement can happen when you start to reduce the time you get with your main interests:

        Whenever I consistently cut quality time for my main interests out of my life, I start to long for them. As you saw in that "typical" day, I do make room for spending time with my family, but my other two main interests are absent. If that happens too many days in a row, I start to really miss reading. I start to really miss playing thoughtful board games with friends. What happens after that? I start to substitute. When I don't have the opportunity to sit down for an hour or even for half an hour and really get lost in a book, I start looking for an alternative way to fill in the tiny slices of time that I do have. I'll spend money.

        You probably have things in your life that provide plenty of gratification, so don't get caught substituting it with impulse buys. Always make sure you keep yourself happy with plenty of time doing the things you like to do and you won't be subconsciously trying to fill that void with useless crap.

        Turn the Money You Save Into More Money

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Once you've programmed your mind to stop buying crap you don't need, you'll have some extra cash to play with. Take all that money and start putting it toward your future and things you will need further down the road. You might need a home, a vehicle, or a way to retire, but none of that can happen until you start planning for it.

        Start by paying off any debts you already have. Credit cards, student loans, and even car payments can force you to live paycheck to paycheck. Use the snowball method and pay off some small balances to make you feel motivated, then start taking out your debt in full force with the stacking method: stop creating new debt, determine which balances have the highest interest rates, and create a payment schedule to pay them off efficiently.

        With your debts whittled down, you should start an emergency fund. No matter how well you plan things, accidents and health emergencies can still happen. An emergency fund is designed to make those kinds of events more manageable. This type of savings account is strictly for when life throws you a curveball, but you can grow one pretty easily with only modest savings.

        When you've paid off your debt and prepared yourself for troubled times, you can start saving for the big stuff. All that money you're not spending on crap anymore can be saved, invested, and compounded to let you buy comfort and security. If you don't know where to start, talk to a financial planner. Or create a simple, yet effective "set and forget" investment portfolio. You've worked hard to reprogram your mind, so make sure you reap the benefits for many years to come.

        Photos by cmgirl (Shutterstock), Macrovector (Shutterstock), J E Theriot, davidd, George Redgrave, David Amsler, Arup Malakar, J B, jakerome, 401(K) 2012.

        ================================================ FILE: test/test-pages/lifehacker-working/source.html ================================================ How to Program Your Mind to Stop Buying Crap You Don’t Need
        Trending on Kinja
        1. 1.
          Aaron Hernandez Is An Idiot, But He Still Might Get Away With Murder
          Aaron Hernandez Is An Idiot, But He Still Might Get Away With Murder
          on Deadspin
        2. 2.
          Top Gear Hosts James May And Richard Hammond React To Clarkson Firing
          on Jalopnik
        3. 3.
          Jeremy Clarkson Officially Fired From Top Gear
          on Jalopnik

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        36
        PatrickAllan

        Patrick Allan

        ProfileFollowUnfollow
        Patrick Allan
        Filed to: money
        3/09/15 8:00am 3/09/15 8:00am

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.

        Understand How Your Own Brain Works Against You

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        It may come as no surprise to learn that stores employ all kinds of tricks to get you to part ways with your cash, and your brain plays right along. Through psychological tricks, product placement, and even color, stores are designed from the ground up to increase spending. We've talked about the biggest things stores do to manipulate your senses, but here are some of the biggest things to look out for:

        • Color: Stores use color to make products attractive and eye-catching, but they also use color on price labels. Red stands out and can encourage taking action, that's why it's commonly associated with sale signage and advertising. When you see red, remember what they're trying to do to your brain with that color. You don't to buy something just because it's on sale.
        • Navigation Roadblocks: Stores force you to walk around stuff you don't need to find the stuff you are really after. Have a list of what you need before you go in, go straight to it, and imagine it's the only item in the store.
        • The Touch Factor: Stores place items they want to sell in easy to reach locations and encourage you to touch them. Don't do it! As soon as you pick something up, you're more likely to buy it because your mind suddenly takes ownership of the object. Don't pick anything up and don't play with display items.
        • Scents and Sounds: You'll probably hear classic, upbeat tunes when you walk into a store. The upbeat music makes you happy and excited, while playing familiar songs makes you feel comfortable. They also use pleasant smells to put your mind at ease. A happy, comfortable mind at ease is a dangerous combination for your brain when shopping. There's not much you can do to avoid this unless you shop online, but it's good to be aware of it.

        And sure, we can blame the stores all we want, but you won't change how they operate—you can only be aware of how your brain is falling for their tricks. Even without the stores, your brain is working against you on its own, thanks to some simple cognitive biases.

        For example, confirmation bias makes you only believe the information that conforms to your prior beliefs, while you discount everything else. Advertisers appeal to this bias directly by convincing you one item is better than another with imagery and other tricks, regardless of what hard facts might say. Keep your mind open, do your own research, and accept when you're wrong about a product. The Decoy effect is also a commonly used tactic. You think one product is a deal because it's next to a similar product that's priced way higher. Even if it's a product you need, it's probably not as good of a deal as it looks right then and there. Again, always research beforehand and be on the lookout for this common trick to avoid impulse buys.

        Make a List of Everything You Own and Do Some Decluttering

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now that you know what you're up against, it's time to start changing the way you think. Before you can stop buying crap you don't need, you need to identify what that crap is. The first step is to make a list of every single thing you own. Every. Single. Thing. This might sound extreme, but you need to gather your data so you can start reprogramming your mind.

        The purpose of this exercise is twofold: you see what you already have and don't need to ever buy again, and you get to see what you shouldn't have bought in the first place. As you list everything out, separate items into categories. It's extremely important that you are as honest with yourself as possible while you do this. It's also important you actually write this all down or type it all out. Here is the first set of categories to separate everything into:

        • Need: You absolutely need this item to get by on a day to day basis.
        • Sometimes Need: You don't need this item every day, but you use it on a somewhat regular basis.
        • Want: You bought this item because you wanted it, not because you needed it.
        • Crap: You don't have a good reason why you have it and you already know it needs to go (there's probably a few of these items, at least).

        Leave the things you listed as "needs" alone, put your stuff listed as "crap" in a pile or box to go bye-bye, and move your attention back to your "sometimes need" and "want" lists. You need to go back over both of those lists because you probably fudged some of the listings, either subconsciously or intentionally. Now ask yourself these three questions as you go through both the "sometimes need" and "want" lists:

        • When was the last time I used this?
        • When will I use this again?
        • Does this item bring you joy?

        Remember to be honest and adjust your lists accordingly. There's nothing wrong with keeping things you wanted. Material items can bring happiness to many people, but make sure the items on your "want" list actively provide you joy and are being used. If an item doesn't get much use or doesn't make you happy, add it to the "crap" list.

        Once you have everything organized, it's time to do some serious decluttering. This listing exercise should get you started, but there are a lot of other great ideas when it comes to ditching the junk you don't need. Regardless, everything on your "crap" list needs to go. You can donate it, sell it at a yard sale, give it away to people know, whatever you like. Before you get rid of everything, though, take a picture of all your stuff together. Print out or save the picture somewhere. Some of it was probably gifts, but in general, this is all the crap you bought that you don't need. Take a good look and remember it.

        See How Much Money and Time You Spent on the Stuff You Threw Out

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now take a look at your "crap" list again and start calculating how much you spent on all of it. If it was a gift, mark it as $0. Otherwise, figure out the price of the item at the time you bought it. If you got a deal or bought it on sale it's okay to factor that in, but try to be as accurate as possible. Once you have the price for each item, add it all together. Depending on your spending habits this could possibly be in the hundreds to thousands of dollars. Remember the picture you took of all this stuff? Attach the total cost to the picture so you can see both at the same time.

        With the money cost figured out, you should take a look at the other costs too. Time is a resource just like any other, and it's a finite one. What kind of time did you pour into these things? Consider the time you spent acquiring and using these items, then write it all down. These can be rough estimations, but go ahead and add it all up when you think you've got it. Now attach the total time to same picture as before and think of the other ways you could have spent all that time. This isn't to make you feel bad about yourself, just to deliver information to your brain in an easy-to-understand form. When you look at it all like this, it can open your eyes a little more, and help you think about purchases in the future. You'll look at an item and ask yourself, "Will this just end up in the picture?"

        List Every Non-Material Thing In Your Life that Makes You Happy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Now it's time to make a different list. While material items may bring plenty of joy, the things in your life that make you happiest probably can't be bought. Get a separate piece of paper or create a new document and list out everything in your life that makes you happy. If you can't buy it, it's eligible for the list. It doesn't matter if it only makes you crack a smile or makes you jump for joy, list it out.

        These are probably the things that actually make you want to get out of bed in the morning and keep on keepin' on. Once you have it all down, put it in your purse or wallet. The next time you feel the urge to buy something, whip this list out first and remind yourself why you probably don't need it.

        Spend Some Time Away from Material Things to Gain Perspective

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you're having a really hard time with your spending, it can help to get away from material objects completely. When you're constantly surrounded by stuff and have access to buying things at all times, it can be really tough to break the habit. Spend a day in the park enjoying the sights and sounds of the outdoors, go camping with some friends, or hike a trail you haven't been on before.

        Essentially, you want to show yourself that you don't need your "things" to have a good time. When you realize how much fun you can have without all the trinkets and trivets, you'll start to shut down your desire to buy them. If you can't get really get away right now, just go for a walk without your purse or wallet (but carry your ID). If you can't buy anything, you'll be forced to experience things a different way.

        Develop a Personal "Should I Buy This?" Test

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        If you don't have a personal "should I buy this?" test, now's the perfect time to make one. When you find an item you think you need or want, it has to pass all of the questions you have on your test before you can buy it. Here's where you can use all of the data you've gathered so far and put it to really good use. The test should be personalized to your own buying habits, but here are some example questions:

        • Is this a planned purchase?
        • Will it end up in the "crap" list picture one day?
        • Where am I going to put it?
        • Have I included this in my budget?
        • Why do I want/need it?

        Custom build your test to hit all of your weaknesses. If you make a lot of impulse buys, include questions that address that. If you experience a lot of buyer's remorse, include a lot of questions that make you think about the use of item after you buy it. If buying the latest and greatest technology is your weakness, Joshua Becker at Becoming Minimalist suggests you ask yourself what problem the piece of tech solves. If you can't think of anything it solves or if you already have something that solves it, you don't need it. Be thorough and build a test that you can run through your mind every time you consider buying something.

        Learn to Delay Gratification and Destroy the Urge to Impulse Buy

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        When it comes to the unnecessary crap we buy, impulse purchases probably make up a good deal of them. We love to feel gratification instantly and impulse buys appeal to that with a rush of excitement with each new purchase. We like to believe that we have control over our impulses all the time, but we really don't, and that's a major problem for the ol' wallet.

        The key is teaching your brain that it's okay to wait for gratification. You can do this with a simple time out every time you want something. Look at whatever you're thinking of buying, go through your personal "should I buy this?" test, and then walk away for a little while. Planning your purchases ahead is ideal, so the longer you can hold off, the better. Set yourself a reminder to check on the item a week or month down the line. When you come back to it, you may find that you don't even want it, just the gratification that would come with it. If you're shopping online, you can do the same thing. Walk away from your desk or put your phone in your pocket and do something else for a little while.

        You can also avoid online impulse purchases by making it harder to do. Block shopping web sites during time periods you know you're at your weakest, or remove all of your saved credit card or Paypal information. You can also practice the "HALT" method when you're shopping online or in a store. Try not to buy things when you're Hungry, Angry, Lonely, or Tired because you're at your weakest state mentally. Last, but not least, the "stranger test" can help you weed out bad purchases too.

        The last thing you should consider when it comes to impulse buys is "artificial replacement." As Trent Hamm at The Simple Dollar explains, artificial replacement can happen when you start to reduce the time you get with your main interests:

        Whenever I consistently cut quality time for my main interests out of my life, I start to long for them. As you saw in that "typical" day, I do make room for spending time with my family, but my other two main interests are absent. If that happens too many days in a row, I start to really miss reading. I start to really miss playing thoughtful board games with friends. What happens after that? I start to substitute. When I don't have the opportunity to sit down for an hour or even for half an hour and really get lost in a book, I start looking for an alternative way to fill in the tiny slices of time that I do have. I'll spend money.

        You probably have things in your life that provide plenty of gratification, so don't get caught substituting it with impulse buys. Always make sure you keep yourself happy with plenty of time doing the things you like to do and you won't be subconsciously trying to fill that void with useless crap.

        Turn the Money You Save Into More Money

        How to Program Your Mind to Stop Buying Crap You Don’t Need

        Once you've programmed your mind to stop buying crap you don't need, you'll have some extra cash to play with. Take all that money and start putting it toward your future and things you will need further down the road. You might need a home, a vehicle, or a way to retire, but none of that can happen until you start planning for it.

        Start by paying off any debts you already have. Credit cards, student loans, and even car payments can force you to live paycheck to paycheck. Use the snowball method and pay off some small balances to make you feel motivated, then start taking out your debt in full force with the stacking method: stop creating new debt, determine which balances have the highest interest rates, and create a payment schedule to pay them off efficiently.

        With your debts whittled down, you should start an emergency fund. No matter how well you plan things, accidents and health emergencies can still happen. An emergency fund is designed to make those kinds of events more manageable. This type of savings account is strictly for when life throws you a curveball, but you can grow one pretty easily with only modest savings.

        When you've paid off your debt and prepared yourself for troubled times, you can start saving for the big stuff. All that money you're not spending on crap anymore can be saved, invested, and compounded to let you buy comfort and security. If you don't know where to start, talk to a financial planner. Or create a simple, yet effective "set and forget" investment portfolio. You've worked hard to reprogram your mind, so make sure you reap the benefits for many years to come.

        Photos by cmgirl (Shutterstock), Macrovector (Shutterstock), J E Theriot, davidd, George Redgrave, David Amsler, Arup Malakar, J B, jakerome, 401(K) 2012.

        36 146Reply
        Patrick Allan’s Discussions
        All replies
        Hosts of other popular chats
        View Patrick Allan's discussionsView all 146 replies
        Powered by

        Kinja

        ================================================ FILE: test/test-pages/links-in-tables/expected-metadata.json ================================================ { "title": "Saving Data: Reducing the size of App Updates by 65%", "byline": null, "dir": "ltr", "excerpt": "Posted by Andrew Hayden, Software Engineer on Google Play Android users are downloading tens of billions of apps and games on Google Pla...", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/links-in-tables/expected.html ================================================

        Posted by Andrew Hayden, Software Engineer on Google Play

        Android users are downloading tens of billions of apps and games on Google Play. We're also seeing developers update their apps frequently in order to provide users with great content, improve security, and enhance the overall user experience. It takes a lot of data to download these updates and we know users care about how much data their devices are using. Earlier this year, we announced that we started using the bsdiff algorithm (by Colin Percival). Using bsdiff, we were able to reduce the size of app updates on average by 47% compared to the full APK size.

        Today, we're excited to share a new approach that goes further — File-by-File patching. App Updates using File-by-File patching are, on average, 65% smaller than the full app, and in some cases more than 90% smaller.

        The savings, compared to our previous approach, add up to 6 petabytes of user data saved per day!

        In order to get the new version of the app, Google Play sends your device a patch that describes the differences between the old and new versions of the app.

        Imagine you are an author of a book about to be published, and wish to change a single sentence - it's much easier to tell the editor which sentence to change and what to change, rather than send an entirely new book. In the same way, patches are much smaller and much faster to download than the entire APK.

        Techniques used in File-by-File patching

        Android apps are packaged as APKs, which are ZIP files with special conventions. Most of the content within the ZIP files (and APKs) is compressed using a technology called Deflate. Deflate is really good at compressing data but it has a drawback: it makes identifying changes in the original (uncompressed) content really hard. Even a tiny change to the original content (like changing one word in a book) can make the compressed output of deflate look completely different. Describing the differences between the original content is easy, but describing the differences between the compressed content is so hard that it leads to inefficient patches.

        Watch how much the compressed text on the right side changes from a one-letter change in the uncompressed text on the left:

        File-by-File therefore is based on detecting changes in the uncompressed data. To generate a patch, we first decompress both old and new files before computing the delta (we still use bsdiff here). Then to apply the patch, we decompress the old file, apply the delta to the uncompressed content and then recompress the new file. In doing so, we need to make sure that the APK on your device is a perfect match, byte for byte, to the one on the Play Store (see APK Signature Schema v2 for why).

        When recompressing the new file, we hit two complications. First, Deflate has a number of settings that affect output; and we don't know which settings were used in the first place. Second, many versions of deflate exist and we need to know whether the version on your device is suitable.

        Fortunately, after analysis of the apps on the Play Store, we've discovered that recent and compatible versions of deflate based on zlib (the most popular deflate library) account for almost all deflated content in the Play Store. In addition, the default settings (level=6) and maximum compression settings (level=9) are the only settings we encountered in practice.

        Knowing this, we can detect and reproduce the original deflate settings. This makes it possible to uncompress the data, apply a patch, and then recompress the data back to exactly the same bytes as originally uploaded.

        However, there is one trade off; extra processing power is needed on the device. On modern devices (e.g. from 2015), recompression can take a little over a second per megabyte and on older or less powerful devices it can be longer. Analysis so far shows that, on average, if the patch size is halved then the time spent applying the patch (which for File-by-File includes recompression) is doubled.

        For now, we are limiting the use of this new patching technology to auto-updates only, i.e. the updates that take place in the background, usually at night when your phone is plugged into power and you're not likely to be using it. This ensures that users won't have to wait any longer than usual for an update to finish when manually updating an app.

        How effective is File-by-File Patching?

        Here are examples of app updates already using File-by-File Patching:

        Application

        Original Size

        Previous (BSDiff) Patch Size

        (% vs original)

        File-by-File Patch Size (% vs original)

        71.1 MB

        13.4 MB (-81%)

        8.0 MB (-89%)

        32.7 MB

        17.5 MB (-46%)

        9.6 MB (-71%)

        17.8 MB

        7.6 MB (-57%)

        7.3 MB (-59%)

        18.9 MB

        17.2 MB (-9%)

        13.1 MB (-31%)

        52.4 MB

        19.1 MB (-64%)

        8.4 MB (-84%)

        16.2 MB

        7.7 MB (-52%)

        1.2 MB (-92%)

        Disclaimer: if you see different patch sizes when you press "update" manually, that is because we are not currently using File-by-file for interactive updates, only those done in the background.

        Saving data and making our users (& developers!) happy

        These changes are designed to ensure our community of over a billion Android users use as little data as possible for regular app updates. The best thing is that as a developer you don't need to do anything. You get these reductions to your update size for free!

        If you'd like to know more about File-by-File patching, including the technical details, head over to the Archive Patcher GitHub project where you can find information, including the source code. Yes, File-by-File patching is completely open-source!

        As a developer if you're interested in reducing your APK size still further, here are some general tips on reducing APK size.

        ================================================ FILE: test/test-pages/links-in-tables/source.html ================================================ Saving Data: Reducing the size of App Updates by 65% | Android Developers Blog
        This Blog
        This Blog
         
         
         

        06 December 2016

        Saving Data: Reducing the size of App Updates by 65%

        Posted by Andrew Hayden, Software Engineer on Google Play

        Android users are downloading tens of billions of apps and games on Google Play. We're also seeing developers update their apps frequently in order to provide users with great content, improve security, and enhance the overall user experience. It takes a lot of data to download these updates and we know users care about how much data their devices are using. Earlier this year, we announced that we started using the bsdiff algorithm (by Colin Percival). Using bsdiff, we were able to reduce the size of app updates on average by 47% compared to the full APK size.

        Today, we're excited to share a new approach that goes further — File-by-File patching. App Updates using File-by-File patching are, on average, 65% smaller than the full app, and in some cases more than 90% smaller.

        The savings, compared to our previous approach, add up to 6 petabytes of user data saved per day!

        In order to get the new version of the app, Google Play sends your device a patch that describes the differences between the old and new versions of the app.

        Imagine you are an author of a book about to be published, and wish to change a single sentence - it's much easier to tell the editor which sentence to change and what to change, rather than send an entirely new book. In the same way, patches are much smaller and much faster to download than the entire APK.

        Techniques used in File-by-File patching

        Android apps are packaged as APKs, which are ZIP files with special conventions. Most of the content within the ZIP files (and APKs) is compressed using a technology called Deflate. Deflate is really good at compressing data but it has a drawback: it makes identifying changes in the original (uncompressed) content really hard. Even a tiny change to the original content (like changing one word in a book) can make the compressed output of deflate look completely different. Describing the differences between the original content is easy, but describing the differences between the compressed content is so hard that it leads to inefficient patches.

        Watch how much the compressed text on the right side changes from a one-letter change in the uncompressed text on the left:

        File-by-File therefore is based on detecting changes in the uncompressed data. To generate a patch, we first decompress both old and new files before computing the delta (we still use bsdiff here). Then to apply the patch, we decompress the old file, apply the delta to the uncompressed content and then recompress the new file. In doing so, we need to make sure that the APK on your device is a perfect match, byte for byte, to the one on the Play Store (see APK Signature Schema v2 for why).

        When recompressing the new file, we hit two complications. First, Deflate has a number of settings that affect output; and we don't know which settings were used in the first place. Second, many versions of deflate exist and we need to know whether the version on your device is suitable.

        Fortunately, after analysis of the apps on the Play Store, we've discovered that recent and compatible versions of deflate based on zlib (the most popular deflate library) account for almost all deflated content in the Play Store. In addition, the default settings (level=6) and maximum compression settings (level=9) are the only settings we encountered in practice.

        Knowing this, we can detect and reproduce the original deflate settings. This makes it possible to uncompress the data, apply a patch, and then recompress the data back to exactly the same bytes as originally uploaded.

        However, there is one trade off; extra processing power is needed on the device. On modern devices (e.g. from 2015), recompression can take a little over a second per megabyte and on older or less powerful devices it can be longer. Analysis so far shows that, on average, if the patch size is halved then the time spent applying the patch (which for File-by-File includes recompression) is doubled.

        For now, we are limiting the use of this new patching technology to auto-updates only, i.e. the updates that take place in the background, usually at night when your phone is plugged into power and you're not likely to be using it. This ensures that users won't have to wait any longer than usual for an update to finish when manually updating an app.

        How effective is File-by-File Patching?

        Here are examples of app updates already using File-by-File Patching:


        Application
        Original Size
        Previous (BSDiff) Patch Size
        (% vs original)
        File-by-File Patch Size (% vs original)
        71.1 MB
        13.4 MB (-81%)
        8.0 MB (-89%)
        32.7 MB
        17.5 MB (-46%)
        9.6 MB (-71%)
        17.8 MB
        7.6 MB (-57%)
        7.3 MB (-59%)
        18.9 MB
        17.2 MB (-9%)
        13.1 MB (-31%)
        52.4 MB
        19.1 MB (-64%)
        8.4 MB (-84%)
        16.2 MB
        7.7 MB (-52%)
        1.2 MB (-92%)


        Disclaimer: if you see different patch sizes when you press "update" manually, that is because we are not currently using File-by-file for interactive updates, only those done in the background.

        Saving data and making our users (& developers!) happy

        These changes are designed to ensure our community of over a billion Android users use as little data as possible for regular app updates. The best thing is that as a developer you don't need to do anything. You get these reductions to your update size for free!

        If you'd like to know more about File-by-File patching, including the technical details, head over to the Archive Patcher GitHub project where you can find information, including the source code. Yes, File-by-File patching is completely open-source!

        As a developer if you're interested in reducing your APK size still further, here are some general tips on reducing APK size.

        Posted by at
        Labels: , , , , ,
        ================================================ FILE: test/test-pages/lwn-1/expected-metadata.json ================================================ { "title": "LWN.net Weekly Edition for March 26, 2015 [LWN.net]", "byline": "By Nathan Willis\n March 25, 2015", "dir": null, "excerpt": "The Arduino has been one of the biggest success stories of the open-hardware movement, but that success does not protect it from internal conflict. In recent months, two of the project's founders have come into conflict about the direction of future efforts—and that conflict has turned into a legal dispute about who owns the rights to the Arduino trademark.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/lwn-1/expected.html ================================================

        A trademark battle in the Arduino community

        The Arduino has been one of the biggest success stories of the open-hardware movement, but that success does not protect it from internal conflict. In recent months, two of the project's founders have come into conflict about the direction of future efforts—and that conflict has turned into a legal dispute about who owns the rights to the Arduino trademark.

        The current fight is a battle between two companies that both bear the Arduino name: Arduino LLC and Arduino SRL. The disagreements that led to present state of affairs go back a bit further.

        The Arduino project grew out of 2005-era course work taught at the Interaction Design Institute Ivrea (IDII) in Ivrea, Italy (using Processing, Wiring, and pre-existing microcontroller hardware). After the IDII program was discontinued, the open-hardware Arduino project as we know it was launched by Massimo Banzi, David Cuartielles, and David Mellis (who had worked together at IDII), with co-founders Tom Igoe and Gianluca Martino joining shortly afterward. The project released open hardware designs (including full schematics and design files) as well as the microcontroller software to run on the boards and the desktop IDE needed to program it.

        Arduino LLC was incorporated in 2008 by Banzi, Cuartielles, Mellis, Igoe, and Martino. The company is registered in the United States, and it has continued to design the Arduino product line, develop the software, and run the Arduino community site. The hardware devices themselves, however, were manufactured by a separate company, "Smart Projects SRL," that was founded by Martino. "SRL" is essentially the Italian equivalent of "LLC"—Smart Projects was incorporated in Italy.

        This division of responsibilities—with the main Arduino project handling everything except for board manufacturing—may seem like an odd one, but it is consistent with Arduino's marketing story. From its earliest days, the designs for the hardware have been freely available, and outside companies were allowed to make Arduino-compatible devices. The project has long run a certification program for third-party manufacturers interested in using the "Arduino" branding, but allows (and arguably even encourages) informal software and firmware compatibility.

        The Arduino branding was not formally registered as a trademark in the early days, however. Arduino LLC filed to register the US trademark in April 2009, and it was granted in 2011.

        At this point, the exact events begin to be harder to verify, but the original group of founders reportedly had a difference of opinion about how to license out hardware production rights to other companies. Wired Italy reports that Martino and Smart Projects resisted the other four founders' plans to "internationalize" production—although it is not clear if that meant that Smart Projects disapproved of licensing out any official hardware manufacturing to other companies, or had some other concern. Heise Online adds that the conflict seemed to be about moving some production to China.

        What is clear is that Smart Projects filed a petition with the US Patent and Trademark Office (USPTO) in October 2014 asking the USPTO to cancel Arduino LLC's trademark on "Arduino." Then, in November 2014, Smart Projects changed its company's name to Arduino SRL. Somewhere around that time, Martino sold off his ownership stake in Smart Projects SRL and new owner Federico Musto was named CEO.

        Unsurprisingly, Arduino LLC did not care for the petition to the USPTO and, in January 2015, the company filed a trademark-infringement lawsuit against Arduino SRL. Confusing matters further, the re-branded Arduino SRL has set up its own web site using the domain name arduino.org, which duplicates most of the site features found on the original Arduino site (arduino.cc). That includes both a hardware store and software downloads.

        Musto, the new CEO of the company now called Arduino SRL, has a bit of a history with Arduino as well. His other manufacturing business had collaborated with Arduino LLC on the design and production of the Arduino Yún, which has received some criticism for including proprietary components.

        Hackaday has run a two-part series (in February and March) digging into the ins and outs of the dispute, including the suggestion that Arduino LLC's recent release of version 1.6.0 of the Arduino IDE was a move intended to block Arduino SRL from hijacking IDE development. Commenter Paul Stoffregen (who was the author of the Heise story above) noted that Arduino SRL recently created a fork of the Arduino IDE on GitHub.

        Most recently, Banzi broke his silence about the dispute in a story published at MAKEzine. There, Banzi claims that Martino secretly filed a trademark application on "Arduino" in Italy in 2008 and told none of the other Arduino founders. He also details a series of unpleasant negotiations between the companies, including Smart Projects stopping the royalty payments it had long sent to Arduino LLC for manufacturing devices and re-branding its boards with the Arduino.org URL.

        Users appear to be stuck in the middle. Banzi says that several retail outlets that claim to be selling "official" Arduino boards are actually paying Arduino SRL, not Arduino LLC, but it is quite difficult to determine which retailers are lined up on which side, since there are (typically) several levels of supplier involved. The two Arduino companies' web sites also disagree about the available hardware, with Arduino.org offering the new Arduino Zero model for sale today and Arduino.cc listing it as "Coming soon."

        Furthermore, as Hackaday's March story explains, the recently-released Arduino.cc IDE now reports that boards manufactured by Arduino SRL are "uncertified." That warning does not prevent users from programming the other company's hardware, but it will no doubt confuse quite a few users who believe they possess genuine Arduino-manufactured devices.

        The USPTO page for Arduino SRL's petition notes pre-trial disclosure dates have been set for August and October of 2015 (for Arduino SRL and Arduino LLC, respectively), which suggests that this debate is far from over. Of course, it is always disappointing to observe a falling out between project founders, particularly when the project in question has had such an impact on open-source software and open hardware.

        One could argue that disputes of this sort are proof that even small projects started among friends need to take legal and intellectual-property issues (such as trademarks) seriously from the very beginning—perhaps Arduino and Smart Projects thought that an informal agreement was all that was necessary in the early days, after all.

        But, perhaps, once a project becomes profitable, there is simply no way to predict what might happen. Arduino LLC would seem to have a strong case for continual and rigorous use of the "Arduino" trademark, which is the salient point in US trademark law. It could still be a while before the courts rule on either side of that question, however.

        Comments (5 posted)

        Mapping and data mining with QGIS 2.8

        By Nathan Willis
        March 25, 2015

        QGIS is a free-software geographic information system (GIS) tool; it provides a unified interface in which users can import, edit, and analyze geographic-oriented information, and it can produce output as varied as printable maps or map-based web services. The project recently made its first update to be designated a long-term release (LTR), and that release is both poised for high-end usage and friendly to newcomers alike.

        The new release is version 2.8, which was unveiled on March 2. An official change log is available on the QGIS site, while the release itself was announced primarily through blog posts (such as this post by Anita Graser of the project's steering committee). Downloads are available for a variety of platforms, including packages for Ubuntu, Debian, Fedora, openSUSE, and several other distributions.

        [QGIS main interface]

        As the name might suggest, QGIS is a Qt application; the latest release will, in fact, build on both Qt4 and Qt5, although the binaries released by the project come only in Qt4 form at present. 2.8 has been labeled a long-term release (LTR)—which, in this case, means that the project has committed to providing backported bug fixes for one full calendar year, and that the 2.8.x series is in permanent feature freeze. The goal, according to the change log, is to provide a stable version suitable for businesses and deployments in other large organizations. The change log itself points out that the development of quite a few new features was underwritten by various GIS companies or university groups, which suggests that taking care of these organizations' needs is reaping dividends for the project.

        For those new to QGIS (or GIS in general), there is a detailed new-user tutorial that provides a thorough walk-through of the data-manipulation, mapping, and analysis functions. Being a new user, I went through the tutorial; although there are a handful of minor differences between QGIS 2.8 and the version used in the text (primarily whether specific features were accessed through a toolbar or right-click menu), on the whole it is well worth the time.

        QGIS is designed to make short work of importing spatially oriented data sets, mining information from them, and turning the results into a meaningful visualization. Technically speaking, the visualization output is optional: one could simply extract the needed statistics and results and use them to answer some question or, perhaps, publish the massaged data set as a database for others to use.

        But well-made maps are often the easiest way to illuminate facts about populations, political regions, geography, and many other topics when human comprehension is the goal. QGIS makes importing data from databases, web-mapping services (WMS), and even unwieldy flat-file data dumps a painless experience. It handles converting between a variety of map-referencing systems more or less automatically, and allows the user to focus on finding the useful attributes of the data sets and rendering them on screen.

        Here be data

        The significant changes in QGIS 2.8 fall into several categories. There are updates to how QGIS handles the mathematical expressions and queries users can use to filter information out of a data set, improvements to the tools used to explore the on-screen map canvas, and enhancements to the "map composer" used to produce visual output. This is on top of plenty of other under-the-hood improvements, naturally.

        [QGIS query builder]

        In the first category are several updates to the filtering tools used to mine a data set. Generally speaking, each independent data set is added to a QGIS project as its own layer, then transformed with filters to focus in on a specific portion of the original data. For instance, the land-usage statistics for a region might be one layer, while roads and buildings for the same region from OpenStreetMap might be two additional layers. Such filters can be created in several ways: there is a "query builder" that lets the user construct and test expressions on a data layer, then save the results, an SQL console for performing similar queries on a database, and spreadsheet-like editing tools for working directly on data tables.

        All three have been improved in this release. New are support for if(condition, true, false) conditional statements, a set of operations for geometry primitives (e.g., to test whether regions overlap or lines intersect), and an "integer divide" operation. Users can also add comments to their queries to annotate their code, and there is a new custom function editor for writing Python functions that can be called in mathematical expressions within the query builder.

        It is also now possible to select only some rows in a table, then perform calculations just on the selection—previously, users would have to extract the rows of interest into a new table first. Similarly, in the SQL editor, the user can highlight a subset of the SQL query and execute it separately, which is no doubt helpful for debugging.

        There have also been several improvements to the Python and Processing plugins. Users can now drag-and-drop Python scripts onto QGIS and they will be run automatically. Several new analysis algorithms are now available through the Processing interface that were previously Python-only; they include algorithms for generating grids of points or vectors within a region, splitting layers and lines, generating hypsometric curves, refactoring data sets, and more.

        Maps in, maps out

        [QGIS simplify tool]

        The process of working with on-screen map data picked up some improvements in the new release as well. Perhaps the most fundamental is that each map layer added to the canvas is now handled in its own thread, so fewer hangs in the user interface are experienced when re-rendering a layer (as happens whenever the user changes the look of points or shapes in a layer). Since remote databases can also be layers, this multi-threaded approach is more resilient against connectivity problems, too. The interface also now supports temporary "scratch" layers that can be used to merge, filter, or simply experiment with a data set, but are not saved when the current project is saved.

        For working on the canvas itself, polygonal regions can now use raster images (tiled, if necessary) as fill colors, the map itself can be rotated arbitrarily, and objects can be "snapped" to align with items on any layer (not just the current layer). For working with raster image layers (e.g., aerial photographs) or simply creating new geometric shapes by hand, there is a new digitizing tool that can offer assistance by locking lines to specific angles, automatically keeping borders parallel, and other niceties.

        There is a completely overhauled "simplify" tool that is used to reduce the number of extraneous vertices of a vector layer (thus reducing its size). The old simplify tool provided only a relative "tolerance" setting that did not correspond directly to any units. With the new tool, users can set a simplification threshold in terms of the underlying map units, layer-specific units, pixels, and more—and, in addition, the tool reports how much the simplify operation has reduced the size of the data.

        [QGIS style editing]

        There has also been an effort to present a uniform interface to one of the most important features of the map canvas: the ability to change the symbology used for an item based on some data attribute. The simplest example might be to change the line color of a road based on whether its road-type attribute is "highway," "service road," "residential," or so on. But the same feature is used to automatically highlight layer information based on the filtering and querying functionality discussed above. The new release allows many more map attributes to be controlled by these "data definition" settings, and provides a hard-to-miss button next to each attribute, through which a custom data definition can be set.

        QGIS's composer module is the tool used to take project data and generate a map that can be used outside of the application (in print, as a static image, or as a layer for MapServer or some other software tool, for example). Consequently, it is not a simple select-and-click-export tool; composing the output can involve a lot of choices about which data to make visible, how (and where) to label it, and how to make it generally accessible.

        The updated composer in 2.8 now has a full-screen mode and sports several new options for configuring output. For instance, the user now has full control over how map axes are labeled. In previous releases, the grid coordinates of the map could be turned on or off, but the only options were all or nothing. Now, the user can individually choose whether coordinates are displayed on all four sides, and can even choose in which direction vertical text labels will run (so that they can be correctly justified to the edge of the map, for example).

        There are, as usual, many more changes than there is room to discuss. Some particularly noteworthy improvements include the ability to save and load bookmarks for frequently used data sources (perhaps most useful for databases, web services, and other non-local data) and improvements to QGIS's server module. This module allows one QGIS instance to serve up data accessible to other QGIS applications (for example, to simply team projects). The server can now be extended with Python plugins and the data layers that it serves can be styled with style rules like those used in the desktop interface.

        QGIS is one of those rare free-software applications that is both powerful enough for high-end work and yet also straightforward to use for the simple tasks that might attract a newcomer to GIS in the first place. The 2.8 release, particularly with its project-wide commitment to long-term support, appears to be an update well worth checking out, whether one needs to create a simple, custom map or to mine a database for obscure geo-referenced meaning.

        Comments (3 posted)

        Development activity in LibreOffice and OpenOffice

        By Jonathan Corbet
        March 25, 2015

        The LibreOffice project was announced with great fanfare in September 2010. Nearly one year later, the OpenOffice.org project (from which LibreOffice was forked) was cut loose from Oracle and found a new home as an Apache project. It is fair to say that the rivalry between the two projects in the time since then has been strong. Predictions that one project or the other would fail have not been borne out, but that does not mean that the two projects are equally successful. A look at the two projects' development communities reveals some interesting differences.

        Release histories

        Apache OpenOffice has made two releases in the past year: 4.1 in April 2014 and 4.1.1 (described as "a micro update" in the release announcement) in August. The main feature added during that time would appear to be significantly improved accessibility support.

        The release history for LibreOffice tells a slightly different story:

        Release Date
        4.2.3 April 2014
        4.1.6 April 2014
        4.2.4 May 2014
        4.2.5 June 2014
        4.3 July 2014
        4.2.6 August 2014
        4.3.1 August 2014
        4.3.2 September 2014
        4.2.7/4.3.3 October 2014
        4.3.4 November 2014
        4.2.8 December 2014
        4.3.5 December 2014
        4.4 January 2015
        4.3.6 February 2015
        4.4.1 February 2015

        It seems clear that LibreOffice has maintained a rather more frenetic release cadence, generally putting out at least one release per month. The project typically keeps at least two major versions alive at any one time. Most of the releases are of the minor, bug-fix variety, but there have been two major releases in the last year as well.

        Development statistics

        In the one-year period since late March 2014, there have been 381 changesets committed to the OpenOffice Subversion repository. The most active committers are:

        Most active OpenOffice developers
        By changesets
        Herbert Dürr 63 16.6%
        Jürgen Schmidt              56 14.7%
        Armin Le Grand 56 14.7%
        Oliver-Rainer Wittmann 46 12.1%
        Tsutomu Uchino 33 8.7%
        Kay Schenk 27 7.1%
        Pedro Giffuni 23 6.1%
        Ariel Constenla-Haile 22 5.8%
        Andrea Pescetti 14 3.7%
        Steve Yin 11 2.9%
        Andre Fischer 10 2.6%
        Yuri Dario 7 1.8%
        Regina Henschel 6 1.6%
        Juan C. Sanz 2 0.5%
        Clarence Guo 2 0.5%
        Tal Daniel 2 0.5%
        By changed lines
        Jürgen Schmidt              455499 88.1%
        Andre Fischer 26148 3.8%
        Pedro Giffuni 23183 3.4%
        Armin Le Grand 11018 1.6%
        Juan C. Sanz 4582 0.7%
        Oliver-Rainer Wittmann 4309 0.6%
        Andrea Pescetti 3908 0.6%
        Herbert Dürr 2811 0.4%
        Tsutomu Uchino 1991 0.3%
        Ariel Constenla-Haile 1258 0.2%
        Steve Yin 1010 0.1%
        Kay Schenk 616 0.1%
        Regina Henschel 417 0.1%
        Yuri Dario 268 0.0%
        tal 16 0.0%
        Clarence Guo 11 0.0%

        In truth, the above list is not just the most active OpenOffice developers — it is all of them; a total of 16 developers have committed changes to OpenOffice in the last year. Those developers changed 528,000 lines of code, but, as can be seen above, Jürgen Schmidt accounted for the bulk of those changes, which were mostly updates to translation files.

        The top four developers in the "by changesets" column all work for IBM, so IBM is responsible for a minimum of about 60% of the changes to OpenOffice in the last year.

        The picture for LibreOffice is just a little bit different; in the same one-year period, the project has committed 22,134 changesets from 268 developers. The most active of these developers were:

        Most active LibreOffice developers
        By changesets
        Caolán McNamara 4307 19.5%
        Stephan Bergmann 2351 10.6%
        Miklos Vajna 1449 6.5%
        Tor Lillqvist 1159 5.2%
        Noel Grandin 1064 4.8%
        Markus Mohrhard 935 4.2%
        Michael Stahl 915 4.1%
        Kohei Yoshida 755 3.4%
        Tomaž Vajngerl 658 3.0%
        Thomas Arnhold 619 2.8%
        Jan Holesovsky 466 2.1%
        Eike Rathke 457 2.1%
        Matteo Casalin 442 2.0%
        Bjoern Michaelsen 421 1.9%
        Chris Sherlock 396 1.8%
        David Tardon 386 1.7%
        Julien Nabet 362 1.6%
        Zolnai Tamás 338 1.5%
        Matúš Kukan 256 1.2%
        Robert Antoni Buj Gelonch 231 1.0%
        By changed lines
        Lionel Elie Mamane 244062 12.5%
        Noel Grandin 238711 12.2%
        Stephan Bergmann 161220 8.3%
        Miklos Vajna 129325 6.6%
        Caolán McNamara 97544 5.0%
        Tomaž Vajngerl 69404 3.6%
        Tor Lillqvist 59498 3.1%
        Laurent Balland-Poirier 52802 2.7%
        Markus Mohrhard 50509 2.6%
        Kohei Yoshida 45514 2.3%
        Chris Sherlock 36788 1.9%
        Peter Foley 34305 1.8%
        Christian Lohmaier 33787 1.7%
        Thomas Arnhold 32722 1.7%
        David Tardon 21681 1.1%
        David Ostrovsky 21620 1.1%
        Jan Holesovsky 20792 1.1%
        Valentin Kettner 20526 1.1%
        Robert Antoni Buj Gelonch 20447 1.0%
        Michael Stahl 18216 0.9%

        To a first approximation, the top ten companies supporting LibreOffice in the last year are:

        Companies supporting LibreOffice development
        (by changesets)
        Red Hat 8417 38.0%
        Collabora Multimedia 6531 29.5%
        (Unknown) 5126 23.2%
        (None) 1490 6.7%
        Canonical 422 1.9%
        Igalia S.L. 80 0.4%
        Ericsson 21 0.1%
        Yandex 18 0.1%
        FastMail.FM 17 0.1%
        SUSE 7 0.0%

        Development work on LibreOffice is thus concentrated in a small number of companies, though it is rather more spread out than OpenOffice development. It is worth noting that the LibreOffice developers with unknown affiliation, who contributed 23% of the changes, make up 82% of the developer base, so there would appear to be a substantial community of developers contributing from outside the above-listed companies.

        Some conclusions

        Last October, some concerns were raised on the OpenOffice list about the health of that project's community. At the time, Rob Weir shrugged them off as the result of a marketing effort by the LibreOffice crowd. There can be no doubt that the war of words between these two projects has gotten tiresome at times, but, looking at the above numbers, it is hard not to conclude that there is an issue that goes beyond marketing hype here.

        In the 4½ years since its founding, the LibreOffice project has put together a community with over 250 active developers. There is support from multiple companies and an impressive rate of patches going into the project's repository. The project's ability to sustain nearly monthly releases on two branches is a direct result of that community's work. Swearing at LibreOffice is one of your editor's favorite pastimes, but it seems clear that the project is on a solid footing with a healthy community.

        OpenOffice, instead, is driven by four developers from a single company — a company that appears to have been deemphasizing OpenOffice work for some time. As a result, the project's commit rate is a fraction of what LibreOffice is able to sustain and releases are relatively rare. As of this writing, the OpenOffice blog shows no posts in 2015. In the October discussion, Rob said that "the dogs may bark but the caravan moves on." That may be true, but, in this case, the caravan does not appear to be moving with any great speed.

        Anything can happen in the free-software development world; it is entirely possible that a reinvigorated OpenOffice.org may yet give LibreOffice a run for its money. But something will clearly have to change to bring that future around. As things stand now, it is hard not to conclude that LibreOffice has won the battle for developer participation.

        Comments (74 posted)

        Page editor: Jonathan Corbet

        Inside this week's LWN.net Weekly Edition

        • Security: Toward secure package downloads; New vulnerabilities in drupal, mozilla, openssl, python-django ...
        • Kernel: LSFMM coverage: NFS, defragmentation, epoll(), copy offload, and more.
        • Distributions: A look at Debian's 2015 DPL candidates; Debian, Fedora, ...
        • Development: A look at GlusterFS; LibreOffice Online; Open sourcing existing code; Secure Boot in Windows 10; ...
        • Announcements: A Turing award for Michael Stonebraker, Sébastien Jodogne, ReGlue are Free Software Award winners, Kat Walsh joins FSF board of directors, Cyanogen, ...

        Next page: Security>>

        ================================================ FILE: test/test-pages/lwn-1/source.html ================================================ LWN.net Weekly Edition for March 26, 2015 [LWN.net]
        LWN.net Logo

        Not logged in

        Log in now

        Create an account

        Subscribe to LWN

        LWN Weekly Edition

        Front page
        Security
        Kernel development
        Distributions
        Development
        Announcements
        ->One big page

         

        This page

        Previous week
        Following week

        Recent Features

        LWN.net Weekly Edition for April 23, 2015

        The kdbuswreck

        LWN.net Weekly Edition for April 16, 2015

        Plotting tools for networks, part I

        Report from the Python Language Summit

        Printable page
        Weekly edition Kernel Security Distributions Contact Us Search
        Archives Calendar Subscribe Write for LWN LWN.net FAQ Sponsors

        LWN.net Weekly Edition for March 26, 2015

        A trademark battle in the Arduino community

        By Nathan Willis
        March 25, 2015

        The Arduino has been one of the biggest success stories of the open-hardware movement, but that success does not protect it from internal conflict. In recent months, two of the project's founders have come into conflict about the direction of future efforts—and that conflict has turned into a legal dispute about who owns the rights to the Arduino trademark.

        The current fight is a battle between two companies that both bear the Arduino name: Arduino LLC and Arduino SRL. The disagreements that led to present state of affairs go back a bit further.

        The Arduino project grew out of 2005-era course work taught at the Interaction Design Institute Ivrea (IDII) in Ivrea, Italy (using Processing, Wiring, and pre-existing microcontroller hardware). After the IDII program was discontinued, the open-hardware Arduino project as we know it was launched by Massimo Banzi, David Cuartielles, and David Mellis (who had worked together at IDII), with co-founders Tom Igoe and Gianluca Martino joining shortly afterward. The project released open hardware designs (including full schematics and design files) as well as the microcontroller software to run on the boards and the desktop IDE needed to program it.

        Arduino LLC was incorporated in 2008 by Banzi, Cuartielles, Mellis, Igoe, and Martino. The company is registered in the United States, and it has continued to design the Arduino product line, develop the software, and run the Arduino community site. The hardware devices themselves, however, were manufactured by a separate company, "Smart Projects SRL," that was founded by Martino. "SRL" is essentially the Italian equivalent of "LLC"—Smart Projects was incorporated in Italy.

        This division of responsibilities—with the main Arduino project handling everything except for board manufacturing—may seem like an odd one, but it is consistent with Arduino's marketing story. From its earliest days, the designs for the hardware have been freely available, and outside companies were allowed to make Arduino-compatible devices. The project has long run a certification program for third-party manufacturers interested in using the "Arduino" branding, but allows (and arguably even encourages) informal software and firmware compatibility.

        The Arduino branding was not formally registered as a trademark in the early days, however. Arduino LLC filed to register the US trademark in April 2009, and it was granted in 2011.

        At this point, the exact events begin to be harder to verify, but the original group of founders reportedly had a difference of opinion about how to license out hardware production rights to other companies. Wired Italy reports that Martino and Smart Projects resisted the other four founders' plans to "internationalize" production—although it is not clear if that meant that Smart Projects disapproved of licensing out any official hardware manufacturing to other companies, or had some other concern. Heise Online adds that the conflict seemed to be about moving some production to China.

        What is clear is that Smart Projects filed a petition with the US Patent and Trademark Office (USPTO) in October 2014 asking the USPTO to cancel Arduino LLC's trademark on "Arduino." Then, in November 2014, Smart Projects changed its company's name to Arduino SRL. Somewhere around that time, Martino sold off his ownership stake in Smart Projects SRL and new owner Federico Musto was named CEO.

        Unsurprisingly, Arduino LLC did not care for the petition to the USPTO and, in January 2015, the company filed a trademark-infringement lawsuit against Arduino SRL. Confusing matters further, the re-branded Arduino SRL has set up its own web site using the domain name arduino.org, which duplicates most of the site features found on the original Arduino site (arduino.cc). That includes both a hardware store and software downloads.

        Musto, the new CEO of the company now called Arduino SRL, has a bit of a history with Arduino as well. His other manufacturing business had collaborated with Arduino LLC on the design and production of the Arduino Yún, which has received some criticism for including proprietary components.

        Hackaday has run a two-part series (in February and March) digging into the ins and outs of the dispute, including the suggestion that Arduino LLC's recent release of version 1.6.0 of the Arduino IDE was a move intended to block Arduino SRL from hijacking IDE development. Commenter Paul Stoffregen (who was the author of the Heise story above) noted that Arduino SRL recently created a fork of the Arduino IDE on GitHub.

        Most recently, Banzi broke his silence about the dispute in a story published at MAKEzine. There, Banzi claims that Martino secretly filed a trademark application on "Arduino" in Italy in 2008 and told none of the other Arduino founders. He also details a series of unpleasant negotiations between the companies, including Smart Projects stopping the royalty payments it had long sent to Arduino LLC for manufacturing devices and re-branding its boards with the Arduino.org URL.

        Users appear to be stuck in the middle. Banzi says that several retail outlets that claim to be selling "official" Arduino boards are actually paying Arduino SRL, not Arduino LLC, but it is quite difficult to determine which retailers are lined up on which side, since there are (typically) several levels of supplier involved. The two Arduino companies' web sites also disagree about the available hardware, with Arduino.org offering the new Arduino Zero model for sale today and Arduino.cc listing it as "Coming soon."

        Furthermore, as Hackaday's March story explains, the recently-released Arduino.cc IDE now reports that boards manufactured by Arduino SRL are "uncertified." That warning does not prevent users from programming the other company's hardware, but it will no doubt confuse quite a few users who believe they possess genuine Arduino-manufactured devices.

        The USPTO page for Arduino SRL's petition notes pre-trial disclosure dates have been set for August and October of 2015 (for Arduino SRL and Arduino LLC, respectively), which suggests that this debate is far from over. Of course, it is always disappointing to observe a falling out between project founders, particularly when the project in question has had such an impact on open-source software and open hardware.

        One could argue that disputes of this sort are proof that even small projects started among friends need to take legal and intellectual-property issues (such as trademarks) seriously from the very beginning—perhaps Arduino and Smart Projects thought that an informal agreement was all that was necessary in the early days, after all.

        But, perhaps, once a project becomes profitable, there is simply no way to predict what might happen. Arduino LLC would seem to have a strong case for continual and rigorous use of the "Arduino" trademark, which is the salient point in US trademark law. It could still be a while before the courts rule on either side of that question, however.

        Comments (5 posted)

        Mapping and data mining with QGIS 2.8

        By Nathan Willis
        March 25, 2015

        QGIS is a free-software geographic information system (GIS) tool; it provides a unified interface in which users can import, edit, and analyze geographic-oriented information, and it can produce output as varied as printable maps or map-based web services. The project recently made its first update to be designated a long-term release (LTR), and that release is both poised for high-end usage and friendly to newcomers alike.

        The new release is version 2.8, which was unveiled on March 2. An official change log is available on the QGIS site, while the release itself was announced primarily through blog posts (such as this post by Anita Graser of the project's steering committee). Downloads are available for a variety of platforms, including packages for Ubuntu, Debian, Fedora, openSUSE, and several other distributions.

        [QGIS main interface]

        As the name might suggest, QGIS is a Qt application; the latest release will, in fact, build on both Qt4 and Qt5, although the binaries released by the project come only in Qt4 form at present. 2.8 has been labeled a long-term release (LTR)—which, in this case, means that the project has committed to providing backported bug fixes for one full calendar year, and that the 2.8.x series is in permanent feature freeze. The goal, according to the change log, is to provide a stable version suitable for businesses and deployments in other large organizations. The change log itself points out that the development of quite a few new features was underwritten by various GIS companies or university groups, which suggests that taking care of these organizations' needs is reaping dividends for the project.

        For those new to QGIS (or GIS in general), there is a detailed new-user tutorial that provides a thorough walk-through of the data-manipulation, mapping, and analysis functions. Being a new user, I went through the tutorial; although there are a handful of minor differences between QGIS 2.8 and the version used in the text (primarily whether specific features were accessed through a toolbar or right-click menu), on the whole it is well worth the time.

        QGIS is designed to make short work of importing spatially oriented data sets, mining information from them, and turning the results into a meaningful visualization. Technically speaking, the visualization output is optional: one could simply extract the needed statistics and results and use them to answer some question or, perhaps, publish the massaged data set as a database for others to use.

        But well-made maps are often the easiest way to illuminate facts about populations, political regions, geography, and many other topics when human comprehension is the goal. QGIS makes importing data from databases, web-mapping services (WMS), and even unwieldy flat-file data dumps a painless experience. It handles converting between a variety of map-referencing systems more or less automatically, and allows the user to focus on finding the useful attributes of the data sets and rendering them on screen.

        Here be data

        The significant changes in QGIS 2.8 fall into several categories. There are updates to how QGIS handles the mathematical expressions and queries users can use to filter information out of a data set, improvements to the tools used to explore the on-screen map canvas, and enhancements to the "map composer" used to produce visual output. This is on top of plenty of other under-the-hood improvements, naturally.

        [QGIS query builder]

        In the first category are several updates to the filtering tools used to mine a data set. Generally speaking, each independent data set is added to a QGIS project as its own layer, then transformed with filters to focus in on a specific portion of the original data. For instance, the land-usage statistics for a region might be one layer, while roads and buildings for the same region from OpenStreetMap might be two additional layers. Such filters can be created in several ways: there is a "query builder" that lets the user construct and test expressions on a data layer, then save the results, an SQL console for performing similar queries on a database, and spreadsheet-like editing tools for working directly on data tables.

        All three have been improved in this release. New are support for if(condition, true, false) conditional statements, a set of operations for geometry primitives (e.g., to test whether regions overlap or lines intersect), and an "integer divide" operation. Users can also add comments to their queries to annotate their code, and there is a new custom function editor for writing Python functions that can be called in mathematical expressions within the query builder.

        It is also now possible to select only some rows in a table, then perform calculations just on the selection—previously, users would have to extract the rows of interest into a new table first. Similarly, in the SQL editor, the user can highlight a subset of the SQL query and execute it separately, which is no doubt helpful for debugging.

        There have also been several improvements to the Python and Processing plugins. Users can now drag-and-drop Python scripts onto QGIS and they will be run automatically. Several new analysis algorithms are now available through the Processing interface that were previously Python-only; they include algorithms for generating grids of points or vectors within a region, splitting layers and lines, generating hypsometric curves, refactoring data sets, and more.

        Maps in, maps out

        [QGIS simplify tool]

        The process of working with on-screen map data picked up some improvements in the new release as well. Perhaps the most fundamental is that each map layer added to the canvas is now handled in its own thread, so fewer hangs in the user interface are experienced when re-rendering a layer (as happens whenever the user changes the look of points or shapes in a layer). Since remote databases can also be layers, this multi-threaded approach is more resilient against connectivity problems, too. The interface also now supports temporary "scratch" layers that can be used to merge, filter, or simply experiment with a data set, but are not saved when the current project is saved.

        For working on the canvas itself, polygonal regions can now use raster images (tiled, if necessary) as fill colors, the map itself can be rotated arbitrarily, and objects can be "snapped" to align with items on any layer (not just the current layer). For working with raster image layers (e.g., aerial photographs) or simply creating new geometric shapes by hand, there is a new digitizing tool that can offer assistance by locking lines to specific angles, automatically keeping borders parallel, and other niceties.

        There is a completely overhauled "simplify" tool that is used to reduce the number of extraneous vertices of a vector layer (thus reducing its size). The old simplify tool provided only a relative "tolerance" setting that did not correspond directly to any units. With the new tool, users can set a simplification threshold in terms of the underlying map units, layer-specific units, pixels, and more—and, in addition, the tool reports how much the simplify operation has reduced the size of the data.

        [QGIS style editing]

        There has also been an effort to present a uniform interface to one of the most important features of the map canvas: the ability to change the symbology used for an item based on some data attribute. The simplest example might be to change the line color of a road based on whether its road-type attribute is "highway," "service road," "residential," or so on. But the same feature is used to automatically highlight layer information based on the filtering and querying functionality discussed above. The new release allows many more map attributes to be controlled by these "data definition" settings, and provides a hard-to-miss button next to each attribute, through which a custom data definition can be set.

        QGIS's composer module is the tool used to take project data and generate a map that can be used outside of the application (in print, as a static image, or as a layer for MapServer or some other software tool, for example). Consequently, it is not a simple select-and-click-export tool; composing the output can involve a lot of choices about which data to make visible, how (and where) to label it, and how to make it generally accessible.

        The updated composer in 2.8 now has a full-screen mode and sports several new options for configuring output. For instance, the user now has full control over how map axes are labeled. In previous releases, the grid coordinates of the map could be turned on or off, but the only options were all or nothing. Now, the user can individually choose whether coordinates are displayed on all four sides, and can even choose in which direction vertical text labels will run (so that they can be correctly justified to the edge of the map, for example).

        There are, as usual, many more changes than there is room to discuss. Some particularly noteworthy improvements include the ability to save and load bookmarks for frequently used data sources (perhaps most useful for databases, web services, and other non-local data) and improvements to QGIS's server module. This module allows one QGIS instance to serve up data accessible to other QGIS applications (for example, to simply team projects). The server can now be extended with Python plugins and the data layers that it serves can be styled with style rules like those used in the desktop interface.

        QGIS is one of those rare free-software applications that is both powerful enough for high-end work and yet also straightforward to use for the simple tasks that might attract a newcomer to GIS in the first place. The 2.8 release, particularly with its project-wide commitment to long-term support, appears to be an update well worth checking out, whether one needs to create a simple, custom map or to mine a database for obscure geo-referenced meaning.

        Comments (3 posted)

        Development activity in LibreOffice and OpenOffice

        By Jonathan Corbet
        March 25, 2015
        The LibreOffice project was announced with great fanfare in September 2010. Nearly one year later, the OpenOffice.org project (from which LibreOffice was forked) was cut loose from Oracle and found a new home as an Apache project. It is fair to say that the rivalry between the two projects in the time since then has been strong. Predictions that one project or the other would fail have not been borne out, but that does not mean that the two projects are equally successful. A look at the two projects' development communities reveals some interesting differences.

        Release histories

        Apache OpenOffice has made two releases in the past year: 4.1 in April 2014 and 4.1.1 (described as "a micro update" in the release announcement) in August. The main feature added during that time would appear to be significantly improved accessibility support.

        The release history for LibreOffice tells a slightly different story:

        Release Date
        4.2.3 April 2014
        4.1.6 April 2014
        4.2.4 May 2014
        4.2.5 June 2014
        4.3 July 2014
        4.2.6 August 2014
        4.3.1 August 2014
        4.3.2 September 2014
        4.2.7/4.3.3 October 2014
        4.3.4 November 2014
        4.2.8 December 2014
        4.3.5 December 2014
        4.4 January 2015
        4.3.6 February 2015
        4.4.1 February 2015

        It seems clear that LibreOffice has maintained a rather more frenetic release cadence, generally putting out at least one release per month. The project typically keeps at least two major versions alive at any one time. Most of the releases are of the minor, bug-fix variety, but there have been two major releases in the last year as well.

        Development statistics

        In the one-year period since late March 2014, there have been 381 changesets committed to the OpenOffice Subversion repository. The most active committers are:

        Most active OpenOffice developers
        By changesets
        Herbert Dürr 63 16.6%
        Jürgen Schmidt              56 14.7%
        Armin Le Grand 56 14.7%
        Oliver-Rainer Wittmann 46 12.1%
        Tsutomu Uchino 33 8.7%
        Kay Schenk 27 7.1%
        Pedro Giffuni 23 6.1%
        Ariel Constenla-Haile 22 5.8%
        Andrea Pescetti 14 3.7%
        Steve Yin 11 2.9%
        Andre Fischer 10 2.6%
        Yuri Dario 7 1.8%
        Regina Henschel 6 1.6%
        Juan C. Sanz 2 0.5%
        Clarence Guo 2 0.5%
        Tal Daniel 2 0.5%
        By changed lines
        Jürgen Schmidt              455499 88.1%
        Andre Fischer 26148 3.8%
        Pedro Giffuni 23183 3.4%
        Armin Le Grand 11018 1.6%
        Juan C. Sanz 4582 0.7%
        Oliver-Rainer Wittmann 4309 0.6%
        Andrea Pescetti 3908 0.6%
        Herbert Dürr 2811 0.4%
        Tsutomu Uchino 1991 0.3%
        Ariel Constenla-Haile 1258 0.2%
        Steve Yin 1010 0.1%
        Kay Schenk 616 0.1%
        Regina Henschel 417 0.1%
        Yuri Dario 268 0.0%
        tal 16 0.0%
        Clarence Guo 11 0.0%

        In truth, the above list is not just the most active OpenOffice developers — it is all of them; a total of 16 developers have committed changes to OpenOffice in the last year. Those developers changed 528,000 lines of code, but, as can be seen above, Jürgen Schmidt accounted for the bulk of those changes, which were mostly updates to translation files.

        The top four developers in the "by changesets" column all work for IBM, so IBM is responsible for a minimum of about 60% of the changes to OpenOffice in the last year.

        The picture for LibreOffice is just a little bit different; in the same one-year period, the project has committed 22,134 changesets from 268 developers. The most active of these developers were:

        Most active LibreOffice developers
        By changesets
        Caolán McNamara 4307 19.5%
        Stephan Bergmann 2351 10.6%
        Miklos Vajna 1449 6.5%
        Tor Lillqvist 1159 5.2%
        Noel Grandin 1064 4.8%
        Markus Mohrhard 935 4.2%
        Michael Stahl 915 4.1%
        Kohei Yoshida 755 3.4%
        Tomaž Vajngerl 658 3.0%
        Thomas Arnhold 619 2.8%
        Jan Holesovsky 466 2.1%
        Eike Rathke 457 2.1%
        Matteo Casalin 442 2.0%
        Bjoern Michaelsen 421 1.9%
        Chris Sherlock 396 1.8%
        David Tardon 386 1.7%
        Julien Nabet 362 1.6%
        Zolnai Tamás 338 1.5%
        Matúš Kukan 256 1.2%
        Robert Antoni Buj Gelonch 231 1.0%
        By changed lines
        Lionel Elie Mamane 244062 12.5%
        Noel Grandin 238711 12.2%
        Stephan Bergmann 161220 8.3%
        Miklos Vajna 129325 6.6%
        Caolán McNamara 97544 5.0%
        Tomaž Vajngerl 69404 3.6%
        Tor Lillqvist 59498 3.1%
        Laurent Balland-Poirier 52802 2.7%
        Markus Mohrhard 50509 2.6%
        Kohei Yoshida 45514 2.3%
        Chris Sherlock 36788 1.9%
        Peter Foley 34305 1.8%
        Christian Lohmaier 33787 1.7%
        Thomas Arnhold 32722 1.7%
        David Tardon 21681 1.1%
        David Ostrovsky 21620 1.1%
        Jan Holesovsky 20792 1.1%
        Valentin Kettner 20526 1.1%
        Robert Antoni Buj Gelonch 20447 1.0%
        Michael Stahl 18216 0.9%

        To a first approximation, the top ten companies supporting LibreOffice in the last year are:

        Companies supporting LibreOffice development
        (by changesets)
        Red Hat 8417 38.0%
        Collabora Multimedia 6531 29.5%
        (Unknown) 5126 23.2%
        (None) 1490 6.7%
        Canonical 422 1.9%
        Igalia S.L. 80 0.4%
        Ericsson 21 0.1%
        Yandex 18 0.1%
        FastMail.FM 17 0.1%
        SUSE 7 0.0%

        Development work on LibreOffice is thus concentrated in a small number of companies, though it is rather more spread out than OpenOffice development. It is worth noting that the LibreOffice developers with unknown affiliation, who contributed 23% of the changes, make up 82% of the developer base, so there would appear to be a substantial community of developers contributing from outside the above-listed companies.

        Some conclusions

        Last October, some concerns were raised on the OpenOffice list about the health of that project's community. At the time, Rob Weir shrugged them off as the result of a marketing effort by the LibreOffice crowd. There can be no doubt that the war of words between these two projects has gotten tiresome at times, but, looking at the above numbers, it is hard not to conclude that there is an issue that goes beyond marketing hype here.

        In the 4½ years since its founding, the LibreOffice project has put together a community with over 250 active developers. There is support from multiple companies and an impressive rate of patches going into the project's repository. The project's ability to sustain nearly monthly releases on two branches is a direct result of that community's work. Swearing at LibreOffice is one of your editor's favorite pastimes, but it seems clear that the project is on a solid footing with a healthy community.

        OpenOffice, instead, is driven by four developers from a single company — a company that appears to have been deemphasizing OpenOffice work for some time. As a result, the project's commit rate is a fraction of what LibreOffice is able to sustain and releases are relatively rare. As of this writing, the OpenOffice blog shows no posts in 2015. In the October discussion, Rob said that "the dogs may bark but the caravan moves on." That may be true, but, in this case, the caravan does not appear to be moving with any great speed.

        Anything can happen in the free-software development world; it is entirely possible that a reinvigorated OpenOffice.org may yet give LibreOffice a run for its money. But something will clearly have to change to bring that future around. As things stand now, it is hard not to conclude that LibreOffice has won the battle for developer participation.

        Comments (74 posted)

        Page editor: Jonathan Corbet

        Inside this week's LWN.net Weekly Edition

        • Security: Toward secure package downloads; New vulnerabilities in drupal, mozilla, openssl, python-django ...
        • Kernel: LSFMM coverage: NFS, defragmentation, epoll(), copy offload, and more.
        • Distributions: A look at Debian's 2015 DPL candidates; Debian, Fedora, ...
        • Development: A look at GlusterFS; LibreOffice Online; Open sourcing existing code; Secure Boot in Windows 10; ...
        • Announcements: A Turing award for Michael Stonebraker, Sébastien Jodogne, ReGlue are Free Software Award winners, Kat Walsh joins FSF board of directors, Cyanogen, ...
        Next page: Security>>

        Copyright © 2015, Eklektix, Inc.
        Comments and public postings are copyrighted by their creators.
        Linux is a registered trademark of Linus Torvalds

        ================================================ FILE: test/test-pages/mathjax/expected-metadata.json ================================================ { "title": "MathJax v3 with MathML input and HTML output", "byline": null, "dir": null, "lang": "en", "excerpt": "When", "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/mathjax/expected.html ================================================

        When

        a ≠ 0 , there are two solutions to a x 2 + b x + c = 0 and they are x = − b ± b 2 − 4 a c 2 a .

        The Lorenz Equations

        x ˙ = σ ( y − x ) y ˙ = ρ x − y − x z z ˙ = − β z + x y

        The Cauchy-Schwarz Inequality

        ( ∑ k = 1 n a k b k ) 2 ≤ ( ∑ k = 1 n a k 2 ) ( ∑ k = 1 n b k 2 )

        A Cross Product Formula

        V 1 × V 2 = | i j k ∂ X ∂ u ∂ Y ∂ u 0 ∂ X ∂ v ∂ Y ∂ v 0 |

        The probability of getting

        k heads when flipping n coins is: P ( E ) = ( n k ) p k ( 1 − p ) n − k

        An Identity of Ramanujan

        1 ( ϕ 5 − ϕ ) e 2 5 π = 1 + e − 2 π 1 + e − 4 π 1 + e − 6 π 1 + e − 8 π 1 + …

        A Rogers-Ramanujan Identity

        1 + q 2 ( 1 − q ) + q 6 ( 1 − q ) ( 1 − q 2 ) + ⋯ = ∏ j = 0 ∞ 1 ( 1 − q 5 j + 2 ) ( 1 − q 5 j + 3 ) , for  | q | < 1 .

        Maxwell's Equations

        ∇ × B → − 1 c ∂ E → ∂ t = 4 π c j → ∇ ⋅ E → = 4 π ρ ∇ × E → + 1 c ∂ B → ∂ t = 0 → ∇ ⋅ B → = 0

        In-line Mathematics

        Finally, while display equations look good for a page of samples, the ability to mix math and text in a paragraph is also important. This expression

        3 x − 1 + ( 1 + x ) 2 is an example of an inline equation. As you see, MathJax equations can be used this way as well, without unduly disturbing the spacing between lines.
        ================================================ FILE: test/test-pages/mathjax/source.html ================================================ MathJax v3 with MathML input and HTML output

        MathJax v3 beta: MathML input, HTML output test

        When

        a ≠ 0 , there are two solutions to a x 2 + b x + c = 0 and they are x = − b ± b 2 − 4 a c 2 a .

        The Lorenz Equations

        x ˙ = σ ( y − x ) y ˙ = ρ x − y − x z z ˙ = − β z + x y

        The Cauchy-Schwarz Inequality

        ( ∑ k = 1 n a k b k ) 2 ≤ ( ∑ k = 1 n a k 2 ) ( ∑ k = 1 n b k 2 )

        A Cross Product Formula

        V 1 × V 2 = | i j k ∂ X ∂ u ∂ Y ∂ u 0 ∂ X ∂ v ∂ Y ∂ v 0 |

        The probability of getting

        k heads when flipping n coins is: P ( E ) = ( n k ) p k ( 1 − p ) n − k

        An Identity of Ramanujan

        1 ( ϕ 5 − ϕ ) e 2 5 π = 1 + e − 2 π 1 + e − 4 π 1 + e − 6 π 1 + e − 8 π 1 + …

        A Rogers-Ramanujan Identity

        1 + q 2 ( 1 − q ) + q 6 ( 1 − q ) ( 1 − q 2 ) + ⋯ = ∏ j = 0 ∞ 1 ( 1 − q 5 j + 2 ) ( 1 − q 5 j + 3 ) , for  | q | < 1 .

        Maxwell's Equations

        ∇ × B → − 1 c ∂ E → ∂ t = 4 π c j → ∇ ⋅ E → = 4 π ρ ∇ × E → + 1 c ∂ B → ∂ t = 0 → ∇ ⋅ B → = 0

        In-line Mathematics

        Finally, while display equations look good for a page of samples, the ability to mix math and text in a paragraph is also important. This expression

        3 x − 1 + ( 1 + x ) 2 is an example of an inline equation. As you see, MathJax equations can be used this way as well, without unduly disturbing the spacing between lines. ================================================ FILE: test/test-pages/medicalnewstoday/expected-metadata.json ================================================ { "title": "How does the brain turn unconscious information into conscious thought?", "byline": "By Ana Sandoiu", "dir": null, "lang": "en", "excerpt": "New research investigates the neurobiological timing of the so-called a-ha! moment that occurs we have come up with the solution to a complex problem.", "siteName": "Medical News Today", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/medicalnewstoday/expected.html ================================================
        Neuroscience tells us that most of the work done by our brains happens on an unconscious level, but when does that "a-ha!" moment occur? And what happens during it? New research investigates.

        hand holding brain lightbulb
        A new study investigates when the 'a-ha!' moment takes place in the brain, and how similar it is to other brain processes.

        Many of us have noticed that we seem to get our best ideas when we're in the shower, or that we can find the answer to a difficult question when we least think about it.

        A large body of neuroscientific studies has pointed out that the brain does a lot of work in its spare time, the so-called idle state - wherein the brain does not appear to be thinking about anything at all - and that this is the time when it works at its hardest to find solutions to complex problems.

        With time and advances in neuroscience, it has become more and more clear to researchers that Freud was right and the mind, as well as the brain, do work unconsciously. In fact, it would be safe to say that what is consciously known to us is just the tip of a much larger iceberg, deeply submerged in unconscious waters.

        But the exact moment at which information becomes known to us - or when the "tip of the iceberg" pierces through the water, and the unconscious becomes conscious - has been somewhat of a mystery, from a neuroscientific point of view.

        In other words, we do not yet know when that intellectually satisfying "a-ha!" moment takes place, or what the biology is behind it. This is why a team of researchers at Columbia University in New York City, NY, set out to investigate this moment in more detail.

        The scientists were led by Michael Shadlen, Ph.D., of Columbia University's Mortimer B. Zuckerman Mind Brain Behavior Institute, and the findings were published in the journal Current Biology.

        The hypothesis

        Dr. Shadlen and colleagues started out from an interesting hypothesis, one which they derived from previous research on the neurobiological processes involved in decision-making.

        As the authors explain, research conducted in both monkeys and humans shows that many of our decisions take place at a point when the brain "feels" as though it has gathered enough information, or when a critical level of information has been accumulated.

        This process of making a decision once the brain has accumulated enough evidence bears the name of "bounded evidence accumulation." Reaching this threshold is important because, although the brain does not use all of the information available, it uses as much as is necessary to make a speedy yet accurate decision.

        The researchers wondered whether or not this threshold is also responsible for our "eureka!" moments.

        In Dr. Shadlen's words, "Could the moment when the brain believes it has accumulated enough evidence be tied to the person's awareness of having decided - that important 'a-ha!' moment?"

        Examining the 'a-ha!' moment

        To answer this question, the scientists asked five people to perform a "direction discrimination" task. In it, the participants looked at dots on a computer screen. The dots moved randomly, as grains of sand would when blown by the wind. The participants were asked to say in which direction the dots had moved.

        The moment they "decided" which direction the dots seemed to be taking was considered to be the equivalent of the "a-ha!" moment.

        In the center of the screen, there was a fixed point and a clock. The display also had two "choice targets" - namely, left or right - and these were the directions in which the participants had to decide that the dots had moved.

        Shortly after the dots had stopped moving, the participants used an electronic, hand-held stylus to move the cursor in the direction that they thought the dots had moved.

        To determine when the decision was made, the researchers used the technique called "mental chronometry" - that is, after they made their decision, the participants were asked to move the clock backward to the point when they felt that they had consciously done so.

        "The moment in time indicated by the participants - this mental chronometry - was entirely subjective; it relied solely on their own estimation of how long it took them to make that decision," Dr. Shadlen says. "And because it was purely subjective, in principle it ought to be unverifiable."

        'A-ha' moment similar to making a decision

        However, by applying a mathematical model, the scientists were able to match these subjective decision times to the bounded evidence accumulation process.

        The subjective decision times fit so well with what the scientists determined as the evidence accumulation threshold that they were able to predict the choices of four of the five participants.

        "If the time reported to us by the participants was valid, we reasoned that it might be possible to predict the accuracy of the decision," explains Dr. Shadlen.

        "We incorporated a kind of mathematical trick, based on earlier studies, which showed that the speed and accuracy of decisions were tied together by the same brain function." This "mathematical trick" was the evidence accumulation model.

        "Essentially, the act of becoming consciously aware of a decision conforms to the same process that the brain goes through to complete a decision, even a simple one - such as whether to turn left or right."

        Michael Shadlen, Ph.D.

        In other words, the study shows that the conscious awareness of the "a-ha!" moment takes place precisely when the brain has reached that threshold of evidence accumulation.

        The findings provide unique insights into the biology of consciousness, say the researchers, and they bring us closer to understanding the biological basis of decisions, ethics, and, generally, the human mind.

        ================================================ FILE: test/test-pages/medicalnewstoday/source.html ================================================ How does the brain turn unconscious information into conscious thought?

        How does the brain turn unconscious information into conscious thought?

        Published Published Thu 27 Jul 2017
        By
        Neuroscience tells us that most of the work done by our brains happens on an unconscious level, but when does that "a-ha!" moment occur? And what happens during it? New research investigates.
        hand holding brain lightbulb
        A new study investigates when the 'a-ha!' moment takes place in the brain, and how similar it is to other brain processes.

        Many of us have noticed that we seem to get our best ideas when we're in the shower, or that we can find the answer to a difficult question when we least think about it.

        A large body of neuroscientific studies has pointed out that the brain does a lot of work in its spare time, the so-called idle state - wherein the brain does not appear to be thinking about anything at all - and that this is the time when it works at its hardest to find solutions to complex problems.

        With time and advances in neuroscience, it has become more and more clear to researchers that Freud was right and the mind, as well as the brain, do work unconsciously. In fact, it would be safe to say that what is consciously known to us is just the tip of a much larger iceberg, deeply submerged in unconscious waters.

        But the exact moment at which information becomes known to us - or when the "tip of the iceberg" pierces through the water, and the unconscious becomes conscious - has been somewhat of a mystery, from a neuroscientific point of view.

        In other words, we do not yet know when that intellectually satisfying "a-ha!" moment takes place, or what the biology is behind it. This is why a team of researchers at Columbia University in New York City, NY, set out to investigate this moment in more detail.

        The scientists were led by Michael Shadlen, Ph.D., of Columbia University's Mortimer B. Zuckerman Mind Brain Behavior Institute, and the findings were published in the journal Current Biology.

        The hypothesis

        Dr. Shadlen and colleagues started out from an interesting hypothesis, one which they derived from previous research on the neurobiological processes involved in decision-making.

        As the authors explain, research conducted in both monkeys and humans shows that many of our decisions take place at a point when the brain "feels" as though it has gathered enough information, or when a critical level of information has been accumulated.

        Scientists find new mechanism for memory formation
        Scientists find new mechanism for memory formation
        How do we form memories? New research uncovers key brain area.
        Read now

        This process of making a decision once the brain has accumulated enough evidence bears the name of "bounded evidence accumulation." Reaching this threshold is important because, although the brain does not use all of the information available, it uses as much as is necessary to make a speedy yet accurate decision.

        The researchers wondered whether or not this threshold is also responsible for our "eureka!" moments.

        In Dr. Shadlen's words, "Could the moment when the brain believes it has accumulated enough evidence be tied to the person's awareness of having decided - that important 'a-ha!' moment?"

        Examining the 'a-ha!' moment

        To answer this question, the scientists asked five people to perform a "direction discrimination" task. In it, the participants looked at dots on a computer screen. The dots moved randomly, as grains of sand would when blown by the wind. The participants were asked to say in which direction the dots had moved.

        The moment they "decided" which direction the dots seemed to be taking was considered to be the equivalent of the "a-ha!" moment.

        In the center of the screen, there was a fixed point and a clock. The display also had two "choice targets" - namely, left or right - and these were the directions in which the participants had to decide that the dots had moved.

        Shortly after the dots had stopped moving, the participants used an electronic, hand-held stylus to move the cursor in the direction that they thought the dots had moved.

        To determine when the decision was made, the researchers used the technique called "mental chronometry" - that is, after they made their decision, the participants were asked to move the clock backward to the point when they felt that they had consciously done so.

        "The moment in time indicated by the participants - this mental chronometry - was entirely subjective; it relied solely on their own estimation of how long it took them to make that decision," Dr. Shadlen says. "And because it was purely subjective, in principle it ought to be unverifiable."

        'A-ha' moment similar to making a decision

        However, by applying a mathematical model, the scientists were able to match these subjective decision times to the bounded evidence accumulation process.

        The subjective decision times fit so well with what the scientists determined as the evidence accumulation threshold that they were able to predict the choices of four of the five participants.

        "If the time reported to us by the participants was valid, we reasoned that it might be possible to predict the accuracy of the decision," explains Dr. Shadlen.

        "We incorporated a kind of mathematical trick, based on earlier studies, which showed that the speed and accuracy of decisions were tied together by the same brain function." This "mathematical trick" was the evidence accumulation model.

        "Essentially, the act of becoming consciously aware of a decision conforms to the same process that the brain goes through to complete a decision, even a simple one - such as whether to turn left or right."

        Michael Shadlen, Ph.D.

        In other words, the study shows that the conscious awareness of the "a-ha!" moment takes place precisely when the brain has reached that threshold of evidence accumulation.

        The findings provide unique insights into the biology of consciousness, say the researchers, and they bring us closer to understanding the biological basis of decisions, ethics, and, generally, the human mind.

        Neurology / Neuroscience
        Biology / Biochemistry Psychology / Psychiatry

        Recommended related news

        Popular in: Neurology / Neuroscience

        TRUSTe
        ================================================ FILE: test/test-pages/medium-1/expected-metadata.json ================================================ { "title": "The Open Journalism Project: Better Student Journalism", "byline": "Pippin Lee", "dir": null, "excerpt": "We pushed out the first version of the Open Journalism site in January. Here’s what we’ve learned about student journali…", "siteName": "Medium", "publishedTime": "2015-03-17T16:27:40.294Z", "readerable": true } ================================================ FILE: test/test-pages/medium-1/expected.html ================================================

        Better Student Journalism

        We pushed out the first version of the Open Journalism site in January. Our goal is for the site to be a place to teach students what they should know about journalism on the web. It should be fun too.

        Topics like mapping, security, command line tools, and open source are all concepts that should be made more accessible, and should be easily understood at a basic level by all journalists. We’re focusing on students because we know student journalism well, and we believe that teaching maturing journalists about the web will provide them with an important lens to view the world with. This is how we got to where we are now.

        Circa 2011

        In late 2011 I sat in the design room of our university’s student newsroom with some of the other editors: Kate Hudson, Brent Rose, and Nicholas Maronese. I was working as the photo editor then—something I loved doing. I was very happy travelling and photographing people while listening to their stories.

        Photography was my lucky way of experiencing the many types of people my generation seemed to avoid, as well as many the public spends too much time discussing. One of my habits as a photographer was scouring sites like Flickr to see how others could frame the world in ways I hadn’t previously considered.

        topleftpixel.com

        I started discovering beautiful things the web could do with images: things not possible with print. Just as every generation revolts against walking in the previous generations shoes, I found myself questioning the expectations that I came up against as a photo editor. In our newsroom the expectations were built from an outdated information world. We were expected to fill old shoes.

        So we sat in our student newsroom—not very happy with what we were doing. Our weekly newspaper had remained essentially unchanged for 40+ years. Each editorial position had the same requirement every year. The big change happened in the 80s when the paper started using colour. We’d also stumbled into having a website, but it was updated just once a week with the release of the newspaper.

        Information had changed form, but the student newsroom hadn’t, and it was becoming harder to romanticize the dusty newsprint smell coming from the shoes we were handed down from previous generations of editors. It was, we were told, all part of “becoming a journalist.”

        We don’t know what we don’t know

        We spent much of the rest of the school year asking “what should we be doing in the newsroom?”, which mainly led us to ask “how do we use the web to tell stories?” It was a straightforward question that led to many more questions about the web: something we knew little about. Out in the real world, traditional journalists were struggling to keep their jobs in a dying print world. They wore the same design of shoes that we were supposed to fill. Being pushed to repeat old, failing strategies and blocked from trying something new scared us.

        We had questions, so we started doing some research. We talked with student newsrooms in Canada and the United States, and filled too many Google Doc files with notes. Looking at the notes now, they scream of fear. We annotated our notes with naive solutions, often involving scrambled and immature odysseys into the future of online journalism.

        There was a lot we didn’t know. We didn’t know how to build a mobile app. We didn’t know if we should build a mobile app. We didn’t know how to run a server. We didn’t know where to go to find a server. We didn’t know how the web worked. We didn’t know how people used the web to read news. We didn’t know what news should be on the web. If news is just information, what does that even look like?

        We asked these questions to many students at other papers to get a consensus of what had worked and what hadn’t. They reported similar questions and fears about the web but followed with “print advertising is keeping us afloat so we can’t abandon it”.

        In other words, we knew that we should be building a newer pair of shoes, but we didn’t know what the function of the shoes should be.

        Common problems in student newsrooms (2011)

        Our questioning of other student journalists in 15 student newsrooms brought up a few repeating issues.

        • Lack of mentorship
        • A news process that lacked consideration of the web
        • No editor/position specific to the web
        • Little exposure to many of the cool projects being put together by professional newsrooms
        • Lack of diverse skills within the newsroom. Writers made up 95% of the personnel. Students with other skills were not sought because journalism was seen as “a career with words.” The other 5% were designers, designing words on computers, for print.
        • Not enough discussion between the business side and web efforts

        From our 2011 research

        Common problems in student newsrooms (2013)

        Two years later, we went back and looked at what had changed. We talked to a dozen more newsrooms and weren’t surprised by our findings.

        • Still no mentorship or link to professional newsrooms building stories for the web
        • Very little control of website and technology
        • The lack of exposure that student journalists have to interactive storytelling. While some newsrooms are in touch with what’s happening with the web and journalism, there still exists a huge gap between the student newsroom and its professional counterpart
        • No time in the current news development cycle for student newsrooms to experiment with the web
        • Lack of skill diversity (specifically coding, interaction design, and statistics)
        • Overly restricted access to student website technology. Changes are primarily visual rather than functional.
        • Significantly reduced print production of many papers
        • Computers aren’t set up for experimenting with software and code, and often locked down

        Newsrooms have traditionally been covered in copies of The New York Times or Globe and Mail. Instead newsrooms should try spend at 20 minutes each week going over the coolest/weirdest online storytelling in an effort to expose each other to what is possible. “Hey, what has the New York Times R&D lab been up to this week?”

        Instead of having computers that are locked down, try setting aside a few office computers that allow students to play and “break”, or encourage editors to buy their own Macbooks so they’re always able to practice with code and new tools on their own.

        From all this we realized that changing a student newsroom is difficult. It takes patience. It requires that the business and editorial departments of the student newsroom be on the same (web)page. The shoes of the future must be different from the shoes we were given.

        We need to rethink how long the new shoe design will be valid. It’s more important that we focus on the process behind making footwear than on actually creating a specific shoe. We shouldn’t be building a shoe to last 40 years. Our footwear design process will allow us to change and adapt as technology evolves. The media landscape will change, so having a newsroom that can change with it will be critical.

        We are building a shoe machine, not a shoe.

        A train or light at the end of the tunnel: are student newsrooms changing for the better?

        In our 2013 research we found that almost 50% of student newsrooms had created roles specifically for the web. This sounds great, but is still problematic in its current state.

        We designed many of these slides to help explain to ourselves what we were doing

        When a newsroom decides to create a position for the web, it’s often with the intent of having content flow steadily from writers onto the web. This is a big improvement from just uploading stories to the web whenever there is a print issue. However…

        1. The handoff
          Problems arise because web editors are given roles that absolve the rest of the editors from thinking about the web. All editors should be involved in the process of story development for the web. While it’s a good idea to have one specific editor manage the website, contributors and editors should all play with and learn about the web. Instead of “can you make a computer do XYZ for me?”, we should be saying “can you show me how to make a computer do XYZ?”
        2. Not just social media
          A web editor could do much more than simply being in charge of the social media accounts for the student paper. Their responsibility could include teaching all other editors to be listening to what’s happening online. The web editor can take advantage of live information to change how the student newsroom reports news in real time.
        3. Web (interactive) editor
          The goal of having a web editor should be for someone to build and tell stories that take full advantage of the web as their medium. Too often the web’s interactivity is not considered when developing the story. The web then ends up as a resting place for print words.

        Editors at newsrooms are still figuring out how to convince writers of the benefit to having their content online. There’s still a stronger draw to writers seeing their name in print than on the web. Showing writers that their stories can be told in new ways to larger audiences is a convincing argument that the web is a starting point for telling a story, not its graveyard.

        When everyone in the newsroom approaches their website with the intention of using it to explore the web as a medium, they all start to ask “what is possible?” and “what can be done?” You can’t expect students to think in terms of the web if it’s treated as a place for print words to hang out on a web page.

        We’re OK with this problem, if we see newsrooms continue to take small steps towards having all their editors involved in the stories for the web.

        The current Open Journalism site was a few years in the making. This was an original launch page we use in 2012

        What we know

        • New process
          Our rough research has told us newsrooms need to be reorganized. This includes every part of the newsroom’s workflow: from where a story and its information comes from, to thinking of every word, pixel, and interaction the reader will have with your stories. If I was a photo editor that wanted to re-think my process with digital tools in mind, I’d start by asking “how are photo assignments processed and sent out?”, “how do we receive images?”, “what formats do images need to be exported in?”, “what type of screens will the images be viewed on?”, and “how are the designers getting these images?” Making a student newsroom digital isn’t about producing “digital manifestos”, it’s about being curious enough that you’ll want to to continue experimenting with your process until you’ve found one that fits your newsroom’s needs.
        • More (remote) mentorship
          Lack of mentorship is still a big problem. Google’s fellowship program is great. The fact that it only caters to United States students isn’t. There are only a handful of internships in Canada where students interested in journalism can get experience writing code and building interactive stories. We’re OK with this for now, as we expect internships and mentorship over the next 5 years between professional newsrooms and student newsrooms will only increase. It’s worth noting that some of that mentorship will likely be done remotely.
        • Changing a newsroom culture
          Skill diversity needs to change. We encourage every student newsroom we talk to, to start building a partnership with their school’s Computer Science department. It will take some work, but you’ll find there are many CS undergrads that love playing with web technologies, and using data to tell stories. Changing who is in the newsroom should be one of the first steps newsrooms take to changing how they tell stories. The same goes with getting designers who understand the wonderful interactive elements of the web and students who love statistics and exploring data. Getting students who are amazing at design, data, code, words, and images into one room is one of the coolest experience I’ve had. Everyone benefits from a more diverse newsroom.

        What we don’t know

        • Sharing curiosity for the web
          We don’t know how to best teach students about the web. It’s not efficient for us to teach coding classes. We do go into newsrooms and get them running their first code exercises, but if someone wants to learn to program, we can only provide the initial push and curiosity. We will be trying out “labs” with a few schools next school year to hopefully get a better idea of how to teach students about the web.
        • Business
          We don’t know how to convince the business side of student papers that they should invest in the web. At the very least we’re able to explain that having students graduate with their current skill set is painful in the current job market.
        • The future
          We don’t know what journalism or the web will be like in 10 years, but we can start encouraging students to keep an open mind about the skills they’ll need. We’re less interested in preparing students for the current newsroom climate, than we are in teaching students to have the ability to learn new tools quickly as they come and go.

        What we’re trying to share with others

        • A concise guide to building stories for the web
          There are too many options to get started. We hope to provide an opinionated guide that follows both our experiences, research, and observations from trying to teach our peers.

        Student newsrooms don’t have investors to please. Student newsrooms can change their website every week if they want to try a new design or interaction. As long as students start treating the web as a different medium, and start building stories around that idea, then we’ll know we’re moving forward.

        A note to professional news orgs

        We’re also asking professional newsrooms to be more open about their process of developing stories for the web. You play a big part in this. This means writing about it, and sharing code. We need to start building a bridge between student journalism and professional newsrooms.

        2012

        This is a start

        We going to continue slowly growing the content on Open Journalism. We still consider this the beta version, but expect to polish it, and beef up the content for a real launch at the beginning of the summer.

        We expect to have more original tutorials as well as the beginnings of what a curriculum may look like that a student newsroom can adopt to start guiding their transition to become a web first newsroom. We’re also going to be working with the Queen’s Journal and The Ubysseynext school year to better understand how to make the student newsroom a place for experimenting with telling stories on the web. If this sound like a good idea in your newsroom, we’re still looking to add 1 more school.

        We’re trying out some new shoes. And while they’re not self-lacing, and smell a bit different, we feel lacing up a new pair of kicks can change a lot.

        Let’s talk. Let’s listen.

        We’re still in the early stages of what this project will look like, so if you want to help or have thoughts, let’s talk.

        pippin@pippinlee.com

        This isn’t supposed to be a manifesto™© we just think it’s pretty cool to share what we’ve learned so far, and hope you’ll do the same. We’re all in this together.

        ================================================ FILE: test/test-pages/medium-1/source.html ================================================ The Open Journalism Project: Better Student Journalism — Medium
        Ready to publish?
        Change the story’s title, subtitle, and visibility as needed
        Pippin Lee
        11 min read

        Open Journalism Project:


        Better Student Journalism




        We pushed out the first version of the Open Journalism site in January. Our goal is for the site to be a place to teach students what they should know about journalism on the web. It should be fun too.

        Topics like mapping, security, command line tools, and open source are all concepts that should be made more accessible, and should be easily understood at a basic level by all journalists. We’re focusing on students because we know student journalism well, and we believe that teaching maturing journalists about the web will provide them with an important lens to view the world with. This is how we got to where we are now.

        Circa 2011

        In late 2011 I sat in the design room of our university’s student newsroom with some of the other editors: Kate Hudson, Brent Rose, and Nicholas Maronese. I was working as the photo editor then—something I loved doing. I was very happy travelling and photographing people while listening to their stories.

        Photography was my lucky way of experiencing the many types of people my generation seemed to avoid, as well as many the public spends too much time discussing. One of my habits as a photographer was scouring sites like Flickr to see how others could frame the world in ways I hadn’t previously considered.

        topleftpixel.com

        I started discovering beautiful things the web could do with images: things not possible with print. Just as every generation revolts against walking in the previous generations shoes, I found myself questioning the expectations that I came up against as a photo editor. In our newsroom the expectations were built from an outdated information world. We were expected to fill old shoes.

        So we sat in our student newsroom—not very happy with what we were doing. Our weekly newspaper had remained essentially unchanged for 40+ years. Each editorial position had the same requirement every year. The big change happened in the 80s when the paper started using colour. We’d also stumbled into having a website, but it was updated just once a week with the release of the newspaper.

        Information had changed form, but the student newsroom hadn’t, and it was becoming harder to romanticize the dusty newsprint smell coming from the shoes we were handed down from previous generations of editors. It was, we were told, all part of “becoming a journalist.”

        We don’t know what we don’t know

        We spent much of the rest of the school year asking “what should we be doing in the newsroom?”, which mainly led us to ask “how do we use the web to tell stories?” It was a straightforward question that led to many more questions about the web: something we knew little about. Out in the real world, traditional journalists were struggling to keep their jobs in a dying print world. They wore the same design of shoes that we were supposed to fill. Being pushed to repeat old, failing strategies and blocked from trying something new scared us.

        We had questions, so we started doing some research. We talked with student newsrooms in Canada and the United States, and filled too many Google Doc files with notes. Looking at the notes now, they scream of fear. We annotated our notes with naive solutions, often involving scrambled and immature odysseys into the future of online journalism.

        There was a lot we didn’t know. We didn’t know how to build a mobile app. We didn’t know if we should build a mobile app. We didn’t know how to run a server. We didn’t know where to go to find a server. We didn’t know how the web worked. We didn’t know how people used the web to read news. We didn’t know what news should be on the web. If news is just information, what does that even look like?

        We asked these questions to many students at other papers to get a consensus of what had worked and what hadn’t. They reported similar questions and fears about the web but followed with “print advertising is keeping us afloat so we can’t abandon it”.

        In other words, we knew that we should be building a newer pair of shoes, but we didn’t know what the function of the shoes should be.

        Common problems in student newsrooms (2011)

        Our questioning of other student journalists in 15 student newsrooms brought up a few repeating issues.

        • Lack of mentorship
        • A news process that lacked consideration of the web
        • No editor/position specific to the web
        • Little exposure to many of the cool projects being put together by professional newsrooms
        • Lack of diverse skills within the newsroom. Writers made up 95% of the personnel. Students with other skills were not sought because journalism was seen as “a career with words.” The other 5% were designers, designing words on computers, for print.
        • Not enough discussion between the business side and web efforts
        From our 2011 research

        Common problems in student newsrooms (2013)

        Two years later, we went back and looked at what had changed. We talked to a dozen more newsrooms and weren’t surprised by our findings.

        • Still no mentorship or link to professional newsrooms building stories for the web
        • Very little control of website and technology
        • The lack of exposure that student journalists have to interactive storytelling. While some newsrooms are in touch with what’s happening with the web and journalism, there still exists a huge gap between the student newsroom and its professional counterpart
        • No time in the current news development cycle for student newsrooms to experiment with the web
        • Lack of skill diversity (specifically coding, interaction design, and statistics)
        • Overly restricted access to student website technology. Changes are primarily visual rather than functional.
        • Significantly reduced print production of many papers
        • Computers aren’t set up for experimenting with software and code, and often locked down

        Newsrooms have traditionally been covered in copies of The New York Times or Globe and Mail. Instead newsrooms should try spend at 20 minutes each week going over the coolest/weirdest online storytelling in an effort to expose each other to what is possible. “Hey, what has the New York Times R&D lab been up to this week?”

        Instead of having computers that are locked down, try setting aside a few office computers that allow students to play and “break”, or encourage editors to buy their own Macbooks so they’re always able to practice with code and new tools on their own.

        From all this we realized that changing a student newsroom is difficult. It takes patience. It requires that the business and editorial departments of the student newsroom be on the same (web)page. The shoes of the future must be different from the shoes we were given.

        We need to rethink how long the new shoe design will be valid. It’s more important that we focus on the process behind making footwear than on actually creating a specific shoe. We shouldn’t be building a shoe to last 40 years. Our footwear design process will allow us to change and adapt as technology evolves. The media landscape will change, so having a newsroom that can change with it will be critical.

        We are building a shoe machine, not a shoe.


        A train or light at the end of the tunnel: are student newsrooms changing for the better?


        In our 2013 research we found that almost 50% of student newsrooms had created roles specifically for the web. This sounds great, but is still problematic in its current state.

        We designed many of these slides to help explain to ourselves what we were doing

        When a newsroom decides to create a position for the web, it’s often with the intent of having content flow steadily from writers onto the web. This is a big improvement from just uploading stories to the web whenever there is a print issue. However…

        1. The handoff
          Problems arise because web editors are given roles that absolve the rest of the editors from thinking about the web. All editors should be involved in the process of story development for the web. While it’s a good idea to have one specific editor manage the website, contributors and editors should all play with and learn about the web. Instead of “can you make a computer do XYZ for me?”, we should be saying “can you show me how to make a computer do XYZ?”
        2. Not just social media
          A web editor could do much more than simply being in charge of the social media accounts for the student paper. Their responsibility could include teaching all other editors to be listening to what’s happening online. The web editor can take advantage of live information to change how the student newsroom reports news in real time.
        3. Web (interactive) editor
          The goal of having a web editor should be for someone to build and tell stories that take full advantage of the web as their medium. Too often the web’s interactivity is not considered when developing the story. The web then ends up as a resting place for print words.

        Editors at newsrooms are still figuring out how to convince writers of the benefit to having their content online. There’s still a stronger draw to writers seeing their name in print than on the web. Showing writers that their stories can be told in new ways to larger audiences is a convincing argument that the web is a starting point for telling a story, not its graveyard.

        When everyone in the newsroom approaches their website with the intention of using it to explore the web as a medium, they all start to ask “what is possible?” and “what can be done?” You can’t expect students to think in terms of the web if it’s treated as a place for print words to hang out on a web page.

        We’re OK with this problem, if we see newsrooms continue to take small steps towards having all their editors involved in the stories for the web.

        The current Open Journalism site was a few years in the making. This was an original launch page we use in 2012

        What we know

        • New process
          Our rough research has told us newsrooms need to be reorganized. This includes every part of the newsroom’s workflow: from where a story and its information comes from, to thinking of every word, pixel, and interaction the reader will have with your stories. If I was a photo editor that wanted to re-think my process with digital tools in mind, I’d start by asking “how are photo assignments processed and sent out?”, “how do we receive images?”, “what formats do images need to be exported in?”, “what type of screens will the images be viewed on?”, and “how are the designers getting these images?” Making a student newsroom digital isn’t about producing “digital manifestos”, it’s about being curious enough that you’ll want to to continue experimenting with your process until you’ve found one that fits your newsroom’s needs.
        • More (remote) mentorship
          Lack of mentorship is still a big problem. Google’s fellowship program is great. The fact that it only caters to United States students isn’t. There are only a handful of internships in Canada where students interested in journalism can get experience writing code and building interactive stories. We’re OK with this for now, as we expect internships and mentorship over the next 5 years between professional newsrooms and student newsrooms will only increase. It’s worth noting that some of that mentorship will likely be done remotely.
        • Changing a newsroom culture
          Skill diversity needs to change. We encourage every student newsroom we talk to, to start building a partnership with their school’s Computer Science department. It will take some work, but you’ll find there are many CS undergrads that love playing with web technologies, and using data to tell stories. Changing who is in the newsroom should be one of the first steps newsrooms take to changing how they tell stories. The same goes with getting designers who understand the wonderful interactive elements of the web and students who love statistics and exploring data. Getting students who are amazing at design, data, code, words, and images into one room is one of the coolest experience I’ve had. Everyone benefits from a more diverse newsroom.

        What we don’t know

        • Sharing curiosity for the web
          We don’t know how to best teach students about the web. It’s not efficient for us to teach coding classes. We do go into newsrooms and get them running their first code exercises, but if someone wants to learn to program, we can only provide the initial push and curiosity. We will be trying out “labs” with a few schools next school year to hopefully get a better idea of how to teach students about the web.
        • Business
          We don’t know how to convince the business side of student papers that they should invest in the web. At the very least we’re able to explain that having students graduate with their current skill set is painful in the current job market.
        • The future
          We don’t know what journalism or the web will be like in 10 years, but we can start encouraging students to keep an open mind about the skills they’ll need. We’re less interested in preparing students for the current newsroom climate, than we are in teaching students to have the ability to learn new tools quickly as they come and go.
        Another slide from 2012 website

        What we’re trying to share with others

        • A concise guide to building stories for the web
          There are too many options to get started. We hope to provide an opinionated guide that follows both our experiences, research, and observations from trying to teach our peers.

        Student newsrooms don’t have investors to please. Student newsrooms can change their website every week if they want to try a new design or interaction. As long as students start treating the web as a different medium, and start building stories around that idea, then we’ll know we’re moving forward.

        A note to professional news orgs

        We’re also asking professional newsrooms to be more open about their process of developing stories for the web. You play a big part in this. This means writing about it, and sharing code. We need to start building a bridge between student journalism and professional newsrooms.

        2012

        This is a start

        We going to continue slowly growing the content on Open Journalism. We still consider this the beta version, but expect to polish it, and beef up the content for a real launch at the beginning of the summer.

        We expect to have more original tutorials as well as the beginnings of what a curriculum may look like that a student newsroom can adopt to start guiding their transition to become a web first newsroom. We’re also going to be working with the Queen’s Journal and The Ubysseynext school year to better understand how to make the student newsroom a place for experimenting with telling stories on the web. If this sound like a good idea in your newsroom, we’re still looking to add 1 more school.

        We’re trying out some new shoes. And while they’re not self-lacing, and smell a bit different, we feel lacing up a new pair of kicks can change a lot.


        Let’s talk. Let’s listen.

        We’re still in the early stages of what this project will look like, so if you want to help or have thoughts, let’s talk.

        pippin@pippinlee.com



        This isn’t supposed to be a manifesto™© we just think it’s pretty cool to share what we’ve learned so far, and hope you’ll do the same. We’re all in this together.

        Written on by
        Pippin Lee
        I don’t know much, so I better start here.
        Thanks to Asad Chishti.
        ================================================ FILE: test/test-pages/medium-2/expected-metadata.json ================================================ { "title": "On Behalf of “Literally”", "byline": "Courtney Kirchoff", "dir": null, "excerpt": "In defense of the word “literally” and why you or someone you know should stop misusing the word, lest they drive us fig…", "siteName": "Medium", "publishedTime": "2015-02-24T19:56:33.374Z", "readerable": true } ================================================ FILE: test/test-pages/medium-2/expected.html ================================================

        Words need defenders.

        On Behalf of “Literally”

        You either are a “literally” abuser or know of one. If you’re anything like me, hearing the word “literally” used incorrectly causes a little piece of your soul to whither and die. Of course I do not mean that literally, I mean that figuratively. An abuser would have said: “Every time a person uses that word, a piece of my soul literally withers and dies.” Which is terribly, horribly wrong.

        For whatever bizarre reason, people feel the need to use literally as a sort of verbal crutch. They use it to emphasize a point, which is silly because they’re already using an analogy or a metaphor to illustrate said point. For example: “Ugh, I literally tore the house apart looking for my remote control!” No, you literally did not tear apart your house, because it’s still standing. If you’d just told me you “tore your house apart” searching for your remote, I would’ve understood what you meant. No need to add “literally” to the sentence.

        Maybe I should define literally.

        Literally means actually. When you say something literally happened, you’re describing the scene or situation as it actually happened.

        So you should only use literally when you mean it. It should not be used in hyperbole. Example: “That was so funny I literally cried.” Which is possible. Some things are funny enough to elicit tears. Note the example stops with “literally cried.” You cannot literally cry your eyes out. The joke wasn’t so funny your eyes popped out of their sockets.

        When in Doubt, Leave it Out

        “I’m so hungry I could eat a horse,” means you’re hungry. You don’t need to say “I’m so hungry I could literally eat a horse.” Because you can’t do that in one sitting, I don’t care how big your stomach is.

        “That play was so funny I laughed my head off,” illustrates the play was amusing. You don’t need to say you literally laughed your head off, because then your head would be on the ground and you wouldn’t be able to speak, much less laugh.

        “I drove so fast my car was flying,” we get your point: you were speeding. But your car is never going fast enough to fly, so don’t say your car was literally flying.

        Insecurities?

        Maybe no one believed a story you told as a child, and you felt the need to prove that it actually happened. No really, mom, I literally climbed the tree. In efforts to prove truth, you used literally to describe something real, however outlandish it seemed. Whatever the reason, now your overuse of literally has become a habit.

        Hard Habit to Break?

        Abusing literally isn’t as bad a smoking, but it’s still an unhealthy habit (I mean that figuratively). Help is required in order to break it.

        This is my version of an intervention for literally abusers. I’m not sure how else to do it other than in writing. I know this makes me sound like a know-it-all, and I accept that. But there’s no excuse other than blatant ignorance to misuse the word “literally.” So just stop it.

        Don’t say “Courtney, this post is so snobbish it literally burned up my computer.” Because nothing is that snobbish that it causes computers to combust. Or: “Courtney, your head is so big it literally cannot get through the door.” Because it can, unless it’s one of those tiny doors from Alice in Wonderland and I need to eat a mushroom to make my whole body smaller.

        No One’s Perfect

        And I’m not saying I am. I’m trying to restore meaning to a word that’s lost meaning. I’m standing up for literally. It’s a good word when used correctly. People are butchering it and destroying it every day (figuratively speaking) and the massacre needs to stop. Just as there’s a coalition of people against the use of certain fonts (like Comic Sans and Papyrus), so should there be a coalition of people against the abuse of literally.

        Saying it to Irritate?

        Do you misuse the word “literally” just to annoy your know-it-all or grammar police friends/acquaintances/total strangers? If so, why? Doing so would be like me going outside when it’s freezing, wearing nothing but a pair of shorts and t-shirt in hopes of making you cold by just looking at me. Who suffers more?

        Graphical Representation

        Matthew Inman of “The Oatmeal” wrote a comic about literally. Abusers and defenders alike should check it out. It’s clear this whole craze about literally is driving a lot of us nuts. You literally abusers are killing off pieces of our souls. You must be stopped, or the world will be lost to meaninglessness forever. Figuratively speaking.

        ================================================ FILE: test/test-pages/medium-2/source.html ================================================ On Behalf of “Literally” — Medium
        sidebar-open-28px
        Ready to publish?
        Change the story’s title, subtitle, and visibility as needed
        Courtney Kirchoff
        4 min read
        Words need defenders.

        On Behalf of “Literally”

        You either are a “literally” abuser or know of one. If you’re anything like me, hearing the word “literally” used incorrectly causes a little piece of your soul to whither and die. Of course I do not mean that literally, I mean that figuratively. An abuser would have said: “Every time a person uses that word, a piece of my soul literally withers and dies.” Which is terribly, horribly wrong.

        For whatever bizarre reason, people feel the need to use literally as a sort of verbal crutch. They use it to emphasize a point, which is silly because they’re already using an analogy or a metaphor to illustrate said point. For example: “Ugh, I literally tore the house apart looking for my remote control!” No, you literally did not tear apart your house, because it’s still standing. If you’d just told me you “tore your house apart” searching for your remote, I would’ve understood what you meant. No need to add “literally” to the sentence.

        Maybe I should define literally.

        Literally means actually. When you say something literally happened, you’re describing the scene or situation as it actually happened.

        So you should only use literally when you mean it. It should not be used in hyperbole. Example: “That was so funny I literally cried.” Which is possible. Some things are funny enough to elicit tears. Note the example stops with “literally cried.” You cannot literally cry your eyes out. The joke wasn’t so funny your eyes popped out of their sockets.

        When in Doubt, Leave it Out

        “I’m so hungry I could eat a horse,” means you’re hungry. You don’t need to say “I’m so hungry I could literally eat a horse.” Because you can’t do that in one sitting, I don’t care how big your stomach is.

        “That play was so funny I laughed my head off,” illustrates the play was amusing. You don’t need to say you literally laughed your head off, because then your head would be on the ground and you wouldn’t be able to speak, much less laugh.

        “I drove so fast my car was flying,” we get your point: you were speeding. But your car is never going fast enough to fly, so don’t say your car was literally flying.

        Insecurities?

        Maybe no one believed a story you told as a child, and you felt the need to prove that it actually happened. No really, mom, I literally climbed the tree. In efforts to prove truth, you used literally to describe something real, however outlandish it seemed. Whatever the reason, now your overuse of literally has become a habit.

        Hard Habit to Break?

        Abusing literally isn’t as bad a smoking, but it’s still an unhealthy habit (I mean that figuratively). Help is required in order to break it.

        This is my version of an intervention for literally abusers. I’m not sure how else to do it other than in writing. I know this makes me sound like a know-it-all, and I accept that. But there’s no excuse other than blatant ignorance to misuse the word “literally.” So just stop it.

        Don’t say “Courtney, this post is so snobbish it literally burned up my computer.” Because nothing is that snobbish that it causes computers to combust. Or: “Courtney, your head is so big it literally cannot get through the door.” Because it can, unless it’s one of those tiny doors from Alice in Wonderland and I need to eat a mushroom to make my whole body smaller.

        No One’s Perfect

        And I’m not saying I am. I’m trying to restore meaning to a word that’s lost meaning. I’m standing up for literally. It’s a good word when used correctly. People are butchering it and destroying it every day (figuratively speaking) and the massacre needs to stop. Just as there’s a coalition of people against the use of certain fonts (like Comic Sans and Papyrus), so should there be a coalition of people against the abuse of literally.

        Saying it to Irritate?

        Do you misuse the word “literally” just to annoy your know-it-all or grammar police friends/acquaintances/total strangers? If so, why? Doing so would be like me going outside when it’s freezing, wearing nothing but a pair of shorts and t-shirt in hopes of making you cold by just looking at me. Who suffers more?

        Graphical Representation

        Matthew Inman of “The Oatmeal” wrote a comic about literally. Abusers and defenders alike should check it out. It’s clear this whole craze about literally is driving a lot of us nuts. You literally abusers are killing off pieces of our souls. You must be stopped, or the world will be lost to meaninglessness forever. Figuratively speaking.

        Originally published at www.courtneykirchoff.com on November 18, 2011. Sadly this solo post did not stop the abuse of literally. Help the word out. Recommend this article to your literally abusers in your life.

        Written on by
        Courtney Kirchoff
        Freelance writer and graphic designer. Will work for bacon. Wrote a novel: Jaden Baker. See my work: www.courtneykirchoff.com & www.truenorthe.com
        ================================================ FILE: test/test-pages/medium-3/expected-metadata.json ================================================ { "title": "Samantha and The Great Big Lie - John C. Welch - Medium", "byline": "John C. Welch", "dir": null, "lang": "en", "excerpt": "(EDIT: removed the link to Samantha’s post, because the arments and the grubers and the rest of The Deck Clique got what they wanted: a non-proper person driven off the internet lightly capped with a…", "siteName": "Medium", "publishedTime": "2015-10-15T02:19:15.607Z", "readerable": true } ================================================ FILE: test/test-pages/medium-3/expected.html ================================================

        John C. Welch

        How to get shanked doing what people say they want

        don’t preach to me
        Mr. integrity

        (EDIT: removed the link to Samantha’s post, because the arments and the grubers and the rest of The Deck Clique got what they wanted: a non-proper person driven off the internet lightly capped with a dusting of transphobia along the way, all totally okay because the ends justify the means, and it’s okay when “good” people do it.)

        First, I need to say something about this article: the reason I’m writing it infuriates me. Worse than installing CS 3 or Acrobat 7 ever did, and the former inspired comparisons to fecophile porn. I’m actually too mad to cuss. Well, not completely, but in this case, I don’t think the people I’m mad at are worth the creativity I try to put into profanity. This is about a brownfield of hypocrisy and viciously deliberate mischaracterization that “shame” cannot even come close to the shame those behind it should feel.

        Now, read this post by Samantha Bielefeld: The Elephant in the Room. First, it is a well-written critical piece that raises a few points in a calm, rational, nonconfrontational fashion, exactly the kind of things the pushers of The Great Big Lie say we need more of, as opposed to the screaming that is the norm in such cases.

        …sorry, I should explain “The Great Big Lie”. There are several, but in this case, our specific instance of “The Great Big Lie” is about criticism. Over and over, you hear from the very people I am not going to be nice to in this that we need “better” criticsm. Instead of rage and anger, volume and vitriol, we need in-depth rational criticism, that isn’t personal or ad hominem. That it should focus on points, not people.

        That, readers, is “The Big Lie”. It is a lie so big that if one ponders the reality of it, as I am going to, one wonders why anyone would believe it. It is a lie and it is one we should stop telling.

        Samantha’s points (I assume you read it, for you are smart people who know the importance of such things) are fairly clear:

        1. With the release of Overcast 2.0, a product Samantha actually likes, Marco Arment moved to a patronage model that will probably be successful for him.
        2. Arment’s insistence that “anyone can do this” while technically true, (anyone can in fact, implement this pricing model), also implies that “anyone” can have the kind of success that a developer with Marco’s history, financial status, and deep ties to the Apple News Web is expected to have. This is silly.
        3. Marco Arment occupies a fairly unique position in the Apple universe, (gained by hard work and no small talent), and because of that, benefits from a set of privileges that a new developer or even one that has been around for a long time, but isn’t, well, Marco, not only don’t have, but have little chance of attaining anytime soon.
        4. Marco has earned his success and is entitled to the benefits and privileges it brings, but he seems rather blind to all of that, and seems to still imagine himself as “two guys in a garage”. This is just not correct.
        5. In addition, the benefits and privileges of the above ensure that by releasing Overcast 2 as a free app, with patronage pricing, he has, if not gutted, severely hurt the ability of folks actually selling their apps for an up-front price of not free to continue doing so. This has the effect of accelerating the “race to the bottom” in the podcast listening app segment, which hurts devs who cannot afford to work on a “I don’t really need this money, so whatever you feel like sending is okay” model.

        None of this is incorrect. None of this is an ad hominem attack in any way. It is just pointing out that a developer of Arment’s stature and status lives in a very different world than someone in East Frog Balls, Arkansas trying to make a living off of App sales. Our dev in EFB doesn’t have the main sites on the Apple web falling all over themselves to review their app the way that Arment does. They’re not friends with the people being The Loop, Daring Fireball, SixColors, iMore, The Mac Observer, etc., yadda.

        So, our hero, in a fit of well-meaning ignorance writes this piece (posted this morning, 14 Oct. 15) and of course, the response and any criticisms are just as reasonable and thoughtful.

        If you really believe that, you are the most preciously ignorant person in the world, and can I have your seriously charmed life.

        The response, from all quarters, including Marco, someone who is so sensitive to criticism that the word “useless” is enough to shut him down, who blocked a friend of mine for the high crime of pointing out that his review of podcasting mics centered around higher priced gear and ignored folks without the scratch, who might not be ready for such things, is, in a single word, disgusting. Vomitous even.

        It’s an hours-long dogpile that beggars even my imagination, and I can imagine almost anything. Seriously, it’s all there in Samantha’s Twitter Feed. From what I can tell, she’s understandably shocked over it. I however was not. This one comment in her feed made me smile (warning, this wanders a bit…er…LOT. Twitter timelines are not easy to put together):

        I can see why you have some reservations about publishing it, but my gut feeling is that he would take it better than Nilay.

        Oh honey, bless your sweet, ignorant heart. Marco is one of the biggest pushers of The Big Lie, and one of the reasons it is such a lie.

        But it gets better. First, you have the “hey, Marco earned his status!” lot. A valid point, and one Bielefeld explicitly acknowledges, here:

        From his ground floor involvement in Tumblr (for which he is now a millionaire), to the creation and sale of a wildly successful app called Instapaper, he has become a household name in technology minded circles. It is this extensive time spent in the spotlight, the huge following on Twitter, and dedicated listeners of his weekly aired Accidental Tech Podcast, that has granted him the freedom to break from seeking revenue in more traditional manners.

        and here:

        I’m not knocking his success, he has put effort into his line of work, and has built his own life.

        and here:

        He has earned his time in the spotlight, and it’s only natural for him to take advantage of it.

        But still, you get the people telling her something she already acknowledge:

        I don’t think he’s blind. he’s worked to where he has gotten and has had failures like everyone else.

        Thank you for restating something in the article. To the person who wrote it.

        In the original article, Samantha talked about the money Marco makes from his podcast. She based that on the numbers provided by ATP in terms of sponsorship rates and the number of current sponsors the podcast has. Is this going to yield perfect numbers? No. But the numbers you get from it will at least be reasonable, or should be unless the published sponsorship rates are just fantasy, and you’re stupid for taking them seriously.

        At first, she went with a simple formula:

        $4K x 3 per episode = $12K x 52 weeks / 3 hosts splitting it.

        That’s not someone making shit up, right? Rather quickly, someone pointed out that she’d made an error in how she calculated it:

        That’s $4k per ad, no? So more like $12–16k per episode.

        She’d already realized her mistake and fixed it.

        which is actually wrong, and I’m correcting now. $4,000 per sponsor, per episode! So, $210,000 per year.

        Again, this is based on publicly available data the only kind someone not part of ATP or a close friend of Arment has access to. So while her numbers may be wrong, if they are, there’s no way for her to know that. She’s basing her opinion on actual available data. Which is sadly rare.

        This becomes a huge flashpoint. You name a reason to attack her over this, people do. No really. For example, she’s not calculating his income taxes correctly:

        especially since it isn’t his only source of income thus, not an indicator of his marginal inc. tax bracket.

        thus, guessing net income is more haphazard than stating approx. gross income.

        Ye Gods. She’s not doing his taxes for him, her point is invalid?

        Then there’s the people who seem to have not read anything past what other people are telling them:

        Not sure what to make of your Marco piece, to be honest. You mention his fame, whatever, but what’s the main idea here?

        Just how spoon-fed do you have to be? Have you no teeth?

        Of course, Marco jumps in, and predictably, he’s snippy:

        If you’re going to speak in precise absolutes, it’s best to first ensure that you’re correct.

        If you’re going to be like that, it’s best to provide better data. Don’t get snippy when someone is going off the only data available, and is clearly open to revising based on better data.

        Then Marco’s friends/fans get into it:

        I really don’t understand why it’s anyone’s business

        Samantha is trying to qualify for sainthood at this point:

        It isn’t really, it was a way of putting his income in context in regards to his ability to gamble with Overcast.

        Again, she’s trying to drag people back to her actual point, but no one is going to play. The storm has begun. Then we get people who are just spouting nonsense:

        Why is that only relevant for him? It’s a pretty weird metric,especially since his apps aren’t free.

        Wha?? Overcast 2 is absolutely free. Samantha points this out:

        His app is free, that’s what sparked the article to begin with.

        The response is literally a parallel to “How can there be global warming if it snowed today in my town?”

        If it’s free, how have I paid for it? Twice?

        She is still trying:

        You paid $4.99 to unlock functionality in Overcast 1.0 and you chose to support him with no additional functionality in 2.0

        He is having none of it. IT SNOWED! SNOWWWWWWW!

        Yes. That’s not free. Free is when you choose not to make money. And that can be weaponized. But that’s not what Overcast does.

        She however, is relentless:

        No, it’s still free. You can choose to support it, you are required to pay $4.99 for Pocket Casts. Totally different model.

        Dude seems to give up. (Note: allllll the people bagging on her are men. All of them. Mansplaining like hell. And I’d bet every one of them considers themselves a feminist.)

        We get another guy trying to push the narrative she’s punishing him for his success, which is just…it’s stupid, okay? Stupid.

        It also wasn’t my point in writing my piece today, but it seems to be everyone’s focus.

        (UNDERSTATEMENT OF THE YEAR)

        I think the focus should be more on that fact that while it’s difficult, Marco spent years building his audience.

        It doesn’t matter what he makes it how he charges. If the audience be earned is willing to pay for it, awesome.

        She tries, oh lord, she tries:

        To assert that he isn’t doing anything any other dev couldn’t, is wrong. It’s successful because it’s Marco.

        But no, HE KNOWS HER POINT BETTER THAN SHE DOES:

        No, it’s successful because he busted his ass to make it so. It’s like any other business. He grew it.

        Christ. This is like a field of strawmen. Stupid ones. Very stupid ones.

        One guy tries to blame it all on Apple, another in a string of Wha??? moments:

        the appropriate context is Apple’s App Store policies. Other devs aren’t Marco’s responsibility

        Seriously? Dude, are you even trying to talk about what Samantha actually wrote? At this point, Samantha is clearly mystified at the entire thing:

        Why has the conversation suddenly turned to focus on nothing more than ATP sponsorship income?

        Because it’s a nit they can pick and allows them to ignore everything you wrote. That’s the only reason.

        One guy is “confused”:

        I see. He does have clout, so are you saying he’s too modest in how he sees himself as a dev?

        Yes. He can’t be equated to the vast majority of other developers. Like calling Gruber, “just another blogger”.

        Alright, that’s fair. I was just confused by the $ and fame angle at first.

        Samantha’s point centers on the benefits Marco gains via his fame and background. HOW DO YOU NOT MENTION THAT? HOW IS THAT CONFUSING?

        People of course are telling her it’s her fault for mentioning a salient fact at all:

        Why has the conversation suddenly turned to focus on nothing more than ATP sponsorship income?

        Maybe because you went there with your article?

        As a way of rationalizing his ability to gamble with the potential for Overcast to generate income…not the norm at all.

        Of course, had she not brought up those important points, she’d have been bagged on for “not providing proof”. Lose some, lose more. By now, she’s had enough and she just deletes all mention of it. Understandable, but sad she was bullied into doing that.

        Yes, bullied. That’s all this is. Bullying. She didn’t lie, cheat, or exaagerate. If her numbers were wrong, they weren’t wrong in a way she had any ability to do anything about. But there’s blood in the water, and the comments and attacks get worse:

        Because you decided to start a conversation about someone else’s personal shit. You started this war.

        War. THIS. IS. WAR.

        This is a bunch of nerds attacking someone for reasoned, calm, polite criticism of their friend/idol. Samantha is politely pushing back a bit:

        That doesn’t explain why every other part of my article is being pushed aside.

        She’s right. This is all nonsense. This is people ignoring her article completely, just looking for things to attack so it can be dismissed. It’s tribalism at its purest.

        Then some of the other annointed get into it, including Jason Snell in one of the most spectactular displays of “I have special knowledge you can’t be expected to have, therefore you are totally off base and wrong, even though there’s no way for you to know this” I’ve seen in a while. Jason:

        You should never use an ad rate card to estimate ad revenue from any media product ever.

        I learned this when I started working for a magazine — rate cards are mostly fiction, like prices on new cars

        How…exactly…in the name of whatever deity Jason may believe in…is Samantha or anyone not “in the biz” supposed to know this. Also, what exactly does a magazine on paper like Macworld have to do with sponsorships for a podcast? I have done podcasts that were sponsored, and I can retaliate with “we charged what the rate card said we did. Checkmate Elitests!”

        Samantha basically abases herself at his feet:

        I understand my mistake, and it’s unfortunate that it has completely diluted the point of my article.

        I think she should have told him where and how to stuff that nonsense, but she’s a nicer person than I am. Also, it’s appropriate that Jason’s twitter avatar has its nose in the air. This is some rank snobbery. It’s disgusting and if anyone pulled that on him, Jason would be very upset. But hey, one cannot criticize The Marco without getting pushback. By “pushback”, I mean “an unrelenting fecal flood”.

        Her only mistake was criticizing one of the Kool Kids. Folks, if you criticize anyone in The Deck Clique, or their friends, expect the same thing, regardless of tone or point.

        Another App Dev, seemingly unable to parse Samantha’s words, needs more explanation:

        so just looking over your mentions, I’m curious what exactly was your main point? Ignoring the podcast income bits.

        Oh wait, he didn’t even read the article. Good on you, Dev Guy, good. on. you. Still, she plays nice with someone who didn’t even read her article:

        That a typical unknown developer can’t depend on patronage to generate revenue, and charging for apps will become a negative.

        Marco comes back of course, and now basically accuses her of lying about other devs talking to her and supporting her point:

        How many actual developers did you hear from, really? Funny how almost nobody wants to give a (real) name on these accusations.

        Really? You’re going to do that? “There’s no name, so I don’t think it’s a real person.” Just…what’s the Joe Welch quote from the McCarthy hearings?

        Let us not assassinate this lad further, Senator. You’ve done enough. Have you no sense of decency, sir? At long last, have you left no sense of decency?

        That is what this is at this point: character assasination because she said something critical of A Popular Person. It’s disgusting. Depressing and disgusting. No one, none of these people have seriously discussed her point, heck, it looks like they barely bothered to read it, if they did at all.

        Marco starts getting really petty with her (no big shock) and Samantha finally starts pushing back:

        Glad to see you be the bigger person and ignore the mindset of so many developers not relating to you, good for you!

        That of course, is what caused Marco to question the validity, if not the existence of her sources. (Funny how anonymous sources are totes okay when they convenience Marco et al, and work for oh, Apple, but when they are inconvenient? Ha! PROVIDE ME PROOF YOU INTEMPERATE WOMAN!)

        Make no mistake, there’s some sexist shit going on here. Every tweet I’ve quoted was authored by a guy.

        Of course, Marco has to play the “I’ve been around longer than you” card with this bon mot:

        Yup, before you existed!

        Really dude? I mean, I’m sorry about the penis, but really?

        Mind you, when the criticism isn’t just bizarrely stupid, Samantha reacts the way Marco and his ilk claim they would to (if they ever got any valid criticism. Which clearly is impossible):

        Not to get into the middle of this, but “income” is not the term you’re looking for. “Revenue” is.

        lol. Noted.

        And I wasn’t intending to be a dick, just a lot of people hear/say “income” when they intend “revenue”, and then discussion …

        … gets derailed by a jedi handwave of “Expenses”. But outside of charitable donation, it is all directly related.

        haha. Thank you for the clarification.

        Note to Marco and the other…whatever they are…that is how one reacts to that kind of criticism. With a bit of humor and self-deprecation. You should try it sometime. For real, not just in your heads or conversations in Irish Pubs in S.F.

        But now, the door has been cracked, and the cheap shots come out:

        @testflight_app: Don’t worry guys, we process @marcoarment’s apps in direct proportion to his megabucks earnings. #fairelephant

        (Note: testflight_app is a parody account. Please do not mess with the actual testflight folks. They are still cool.)

        Or this…conversation:

        Image for post

        Good job guys. Good job. Defend the tribe. Attack the other. Frederico attempts to recover from his stunning display of demeaning douchery: ‏@viticci: @s_bielefeld I don’t know if it’s an Italian thing, but counting other people’s money is especially weird for me. IMO, bad move in the post.

        Samantha is clearly sick of his crap: ‏@s_bielefeld: @viticci That’s what I’m referring to, the mistake of ever having mentioned it. So, now, Marco can ignore the bigger issue and go on living.

        Good for her. There’s being patient and being roadkill.

        Samantha does put the call out for her sources to maybe let her use their names:

        From all of you I heard from earlier, anyone care to go on record?

        My good friend, The Angry Drunk points out the obvious problem:

        Nobody’s going to go on record when they count on Marco’s friends for their PR.

        This is true. Again, the sites that are Friends of Marco:

        Daring Fireball

        The Loop

        SixColors

        iMore

        MacStories

        A few others, but I want this post to end one day.

        You piss that crew off, and given how petty rather a few of them have demonstrated they are, good luck on getting any kind of notice from them.

        Of course, the idea this could happen is just craycray:

        @KevinColeman .@Angry_Drunk @s_bielefeld @marcoarment Wow, you guys are veering right into crazy conspiracy theory territory. #JetFuelCantMeltSteelBeams

        Yeah. Because a mature person like Marco would never do anything like that.

        Of course, the real point on this is starting to happen:

        you’re getting a lot of heat now but happy you are writing things that stir up the community. Hope you continue to be a voice!

        I doubt I will.

        See, they’ve done their job. Mess with the bull, you get the horns. Maybe you should find another thing to write about, this isn’t a good place for you. Great job y’all.

        Some people aren’t even pretending. They’re just in full strawman mode:

        @timkeller: Unfair to begrudge a person for leveraging past success, especially when that success is earned. No ‘luck’ involved.

        @s_bielefeld: @timkeller I plainly stated that I don’t hold his doing this against him. Way to twist words.

        I think she’s earned her anger at this point.

        Don’t worry, Marco knows what the real problem is: most devs just suck —

        Image for post

        I have a saying that applies in this case: don’t place your head so far up your nethers that you go full Klein Bottle. Marco has gone full Klein Bottle. (To be correct, he went FKB some years ago.)

        There are some bright spots. My favorite is when Building Twenty points out the real elephant in the room:

        @BuildingTwenty: Both @s_bielefeld & I wrote similar critiques of @marcoarment’s pricing model yet the Internet pilloried only the woman. Who’d have guessed?

        Yup.

        Another bright spot are these comments from Ian Betteridge, who has been doing this even longer than Marco:

        You know, any writer who has never made a single factual error in a piece hasn’t ever written anything worth reading.

        I learned my job with the support of people who helped me. Had I suffered an Internet pile on for every error I wouldn’t have bothered.

        To which Samantha understandably replies:

        and it’s honestly something I’m contemplating right now, whether to continue…

        Gee, I can’t imagine why. Why with comments like this from Chris Breen that completely misrepresent Samantha’s point, (who until today, I would have absolutely defended as being better than this, something I am genuinely saddened to be wrong about), why wouldn’t she want to continue doing this?

        If I have this right, some people are outraged that a creator has decided to give away his work.

        No Chris, you don’t have this right. But hey, who has time to find out the real issue and read an article. I’m sure your friends told you everything you need to know.

        Noted Feminist Glenn Fleishman gets a piece of the action too:

        Image for post

        I’m not actually surprised here. I watched Fleishman berate a friend of mine who has been an engineer for…heck, waaaaay too long on major software products in the most condescending way because she tried to point out that as a very technical woman, “The Magazine” literally had nothing to say to her and maybe he should fix that. “Impertinent” was I believe what he called her, but I may have the specific word wrong. Not the attitude mind you. Great Feminists like Glenn do not like uppity women criticizing Great Feminists who are their Great Allies.

        Great Feminists are often tools.

        Luckily, I hope, the people who get Samantha’s point also started chiming in (and you get 100% of the women commenting here that I’ve seen):

        I don’t think he’s wrong for doing it, he just discusses it as if the market’s a level playing field — it isn’t

        This is a great article with lots of great points about the sustainability of iOS development. Thank you for publishing it.

        Regardless of the numbers and your view of MA, fair points here about confirmation bias in app marketing feasibility http://samanthabielefeld.com/the-elephant-in-the-room …

        thank you for posting this, it covers a lot of things people don’t like to talk about.

        I’m sure you have caught untold amounts of flak over posting this because Marco is blind to his privilege as a developer.

        Catching up on the debate, and agreeing with Harry’s remark. (Enjoyed your article, Samantha, and ‘got’ your point.)

        I would like to say I’m surprised at the reaction to Samantha’s article, but I’m not. In spite of his loud declarations of support for The Big Lie, Marco Arment is as bad at any form of criticism that he hasn’t already approved as a very insecure tween. An example from 2011: http://www.businessinsider.com/marco-arment-2011-9

        Marco is great with criticism as long as it never actually criticizes him. If it does, be prepared a flood of petty, petulant whining that a room full of bored preschoolers on a hot day would be hard-pressed to match.

        Today has been…well, it sucks. It sucks because someone doing what all the Arments of the world claim to want was naive enough to believe what they were told, and found out the hard way just how big a lie The Big Lie is, and how vicious people are when you’re silly enough to believe anything they say about criticism.

        And note again, every single condescending crack, misrepresentation, and strawman had an exclusively male source. Most of them have, at one point or another, loudly trumpted themselves as Feminist Allies, as a friend to women struggling with the sexism and misogyny in tech. Congratulations y’all on being just as bad as the people you claim to oppose.

        Samantha has handled this better than anyone else could have. My respect for her as a person and a writer is off the charts. If she choses to walk away from blogging in the Apple space, believe me I understand. As bad as today was for her, I’ve seen worse. Much worse.

        But I hope she doesn’t. I hope she stays, because she is Doing This Right, and in a corner of the internet that has become naught but an endless circle jerk, a cliquish collection, a churlish, childish cohort interested not in writing or the truth, but in making sure The Right People are elevated, and The Others put down, she is someone worth reading and listening to. The number people who owe her apologies goes around the block, and I don’t think she’ll ever see a one. I’m sure as heck not apologizing for them, I’ll not make their lives easier in the least.

        All of you, all. of. you…Marco, Breen, Snell, Vittici, had a chance to live by your words. You were faced with reasoned, polite, respectful criticism and instead of what you should have done, you all dropped trou and sprayed an epic diarrheal discharge all over someone who had done nothing to deserve it. Me, I earned most of my aggro, Samantha did not earn any of the idiocy I’ve seen today. I hope you’re all proud of yourselves. Someone should be, it won’t be me. Ever.

        So I hope she stays, but if she goes, I understand. For what it’s worth, I don’t think she’s wrong either way.

        ================================================ FILE: test/test-pages/medium-3/source.html ================================================ Samantha and The Great Big Lie. How to get shanked doing what people… | by John C. Welch | Medium

        Samantha and The Great Big Lie

        John C. Welch
        John C. Welch
        Oct 15, 2015 · 18 min read

        How to get shanked doing what people say they want

        don’t preach to me
        Mr. integrity

        (EDIT: removed the link to Samantha’s post, because the arments and the grubers and the rest of The Deck Clique got what they wanted: a non-proper person driven off the internet lightly capped with a dusting of transphobia along the way, all totally okay because the ends justify the means, and it’s okay when “good” people do it.)

        First, I need to say something about this article: the reason I’m writing it infuriates me. Worse than installing CS 3 or Acrobat 7 ever did, and the former inspired comparisons to fecophile porn. I’m actually too mad to cuss. Well, not completely, but in this case, I don’t think the people I’m mad at are worth the creativity I try to put into profanity. This is about a brownfield of hypocrisy and viciously deliberate mischaracterization that “shame” cannot even come close to the shame those behind it should feel.

        Now, read this post by Samantha Bielefeld: The Elephant in the Room. First, it is a well-written critical piece that raises a few points in a calm, rational, nonconfrontational fashion, exactly the kind of things the pushers of The Great Big Lie say we need more of, as opposed to the screaming that is the norm in such cases.

        …sorry, I should explain “The Great Big Lie”. There are several, but in this case, our specific instance of “The Great Big Lie” is about criticism. Over and over, you hear from the very people I am not going to be nice to in this that we need “better” criticsm. Instead of rage and anger, volume and vitriol, we need in-depth rational criticism, that isn’t personal or ad hominem. That it should focus on points, not people.

        That, readers, is “The Big Lie”. It is a lie so big that if one ponders the reality of it, as I am going to, one wonders why anyone would believe it. It is a lie and it is one we should stop telling.

        Samantha’s points (I assume you read it, for you are smart people who know the importance of such things) are fairly clear:

        1. With the release of Overcast 2.0, a product Samantha actually likes, Marco Arment moved to a patronage model that will probably be successful for him.
        2. Arment’s insistence that “anyone can do this” while technically true, (anyone can in fact, implement this pricing model), also implies that “anyone” can have the kind of success that a developer with Marco’s history, financial status, and deep ties to the Apple News Web is expected to have. This is silly.
        3. Marco Arment occupies a fairly unique position in the Apple universe, (gained by hard work and no small talent), and because of that, benefits from a set of privileges that a new developer or even one that has been around for a long time, but isn’t, well, Marco, not only don’t have, but have little chance of attaining anytime soon.
        4. Marco has earned his success and is entitled to the benefits and privileges it brings, but he seems rather blind to all of that, and seems to still imagine himself as “two guys in a garage”. This is just not correct.
        5. In addition, the benefits and privileges of the above ensure that by releasing Overcast 2 as a free app, with patronage pricing, he has, if not gutted, severely hurt the ability of folks actually selling their apps for an up-front price of not free to continue doing so. This has the effect of accelerating the “race to the bottom” in the podcast listening app segment, which hurts devs who cannot afford to work on a “I don’t really need this money, so whatever you feel like sending is okay” model.

        None of this is incorrect. None of this is an ad hominem attack in any way. It is just pointing out that a developer of Arment’s stature and status lives in a very different world than someone in East Frog Balls, Arkansas trying to make a living off of App sales. Our dev in EFB doesn’t have the main sites on the Apple web falling all over themselves to review their app the way that Arment does. They’re not friends with the people being The Loop, Daring Fireball, SixColors, iMore, The Mac Observer, etc., yadda.

        So, our hero, in a fit of well-meaning ignorance writes this piece (posted this morning, 14 Oct. 15) and of course, the response and any criticisms are just as reasonable and thoughtful.

        If you really believe that, you are the most preciously ignorant person in the world, and can I have your seriously charmed life.

        The response, from all quarters, including Marco, someone who is so sensitive to criticism that the word “useless” is enough to shut him down, who blocked a friend of mine for the high crime of pointing out that his review of podcasting mics centered around higher priced gear and ignored folks without the scratch, who might not be ready for such things, is, in a single word, disgusting. Vomitous even.

        It’s an hours-long dogpile that beggars even my imagination, and I can imagine almost anything. Seriously, it’s all there in Samantha’s Twitter Feed. From what I can tell, she’s understandably shocked over it. I however was not. This one comment in her feed made me smile (warning, this wanders a bit…er…LOT. Twitter timelines are not easy to put together):

        I can see why you have some reservations about publishing it, but my gut feeling is that he would take it better than Nilay.

        Oh honey, bless your sweet, ignorant heart. Marco is one of the biggest pushers of The Big Lie, and one of the reasons it is such a lie.

        But it gets better. First, you have the “hey, Marco earned his status!” lot. A valid point, and one Bielefeld explicitly acknowledges, here:

        From his ground floor involvement in Tumblr (for which he is now a millionaire), to the creation and sale of a wildly successful app called Instapaper, he has become a household name in technology minded circles. It is this extensive time spent in the spotlight, the huge following on Twitter, and dedicated listeners of his weekly aired Accidental Tech Podcast, that has granted him the freedom to break from seeking revenue in more traditional manners.

        and here:

        I’m not knocking his success, he has put effort into his line of work, and has built his own life.

        and here:

        He has earned his time in the spotlight, and it’s only natural for him to take advantage of it.

        But still, you get the people telling her something she already acknowledge:

        I don’t think he’s blind. he’s worked to where he has gotten and has had failures like everyone else.

        Thank you for restating something in the article. To the person who wrote it.

        In the original article, Samantha talked about the money Marco makes from his podcast. She based that on the numbers provided by ATP in terms of sponsorship rates and the number of current sponsors the podcast has. Is this going to yield perfect numbers? No. But the numbers you get from it will at least be reasonable, or should be unless the published sponsorship rates are just fantasy, and you’re stupid for taking them seriously.

        At first, she went with a simple formula:

        $4K x 3 per episode = $12K x 52 weeks / 3 hosts splitting it.

        That’s not someone making shit up, right? Rather quickly, someone pointed out that she’d made an error in how she calculated it:

        That’s $4k per ad, no? So more like $12–16k per episode.

        She’d already realized her mistake and fixed it.

        which is actually wrong, and I’m correcting now. $4,000 per sponsor, per episode! So, $210,000 per year.

        Again, this is based on publicly available data the only kind someone not part of ATP or a close friend of Arment has access to. So while her numbers may be wrong, if they are, there’s no way for her to know that. She’s basing her opinion on actual available data. Which is sadly rare.

        This becomes a huge flashpoint. You name a reason to attack her over this, people do. No really. For example, she’s not calculating his income taxes correctly:

        especially since it isn’t his only source of income thus, not an indicator of his marginal inc. tax bracket.

        thus, guessing net income is more haphazard than stating approx. gross income.

        Ye Gods. She’s not doing his taxes for him, her point is invalid?

        Then there’s the people who seem to have not read anything past what other people are telling them:

        Not sure what to make of your Marco piece, to be honest. You mention his fame, whatever, but what’s the main idea here?

        Just how spoon-fed do you have to be? Have you no teeth?

        Of course, Marco jumps in, and predictably, he’s snippy:

        If you’re going to speak in precise absolutes, it’s best to first ensure that you’re correct.

        If you’re going to be like that, it’s best to provide better data. Don’t get snippy when someone is going off the only data available, and is clearly open to revising based on better data.

        Then Marco’s friends/fans get into it:

        I really don’t understand why it’s anyone’s business

        Samantha is trying to qualify for sainthood at this point:

        It isn’t really, it was a way of putting his income in context in regards to his ability to gamble with Overcast.

        Again, she’s trying to drag people back to her actual point, but no one is going to play. The storm has begun. Then we get people who are just spouting nonsense:

        Why is that only relevant for him? It’s a pretty weird metric,especially since his apps aren’t free.

        Wha?? Overcast 2 is absolutely free. Samantha points this out:

        His app is free, that’s what sparked the article to begin with.

        The response is literally a parallel to “How can there be global warming if it snowed today in my town?”

        If it’s free, how have I paid for it? Twice?

        She is still trying:

        You paid $4.99 to unlock functionality in Overcast 1.0 and you chose to support him with no additional functionality in 2.0

        He is having none of it. IT SNOWED! SNOWWWWWWW!

        Yes. That’s not free. Free is when you choose not to make money. And that can be weaponized. But that’s not what Overcast does.

        She however, is relentless:

        No, it’s still free. You can choose to support it, you are required to pay $4.99 for Pocket Casts. Totally different model.

        Dude seems to give up. (Note: allllll the people bagging on her are men. All of them. Mansplaining like hell. And I’d bet every one of them considers themselves a feminist.)

        We get another guy trying to push the narrative she’s punishing him for his success, which is just…it’s stupid, okay? Stupid.

        It also wasn’t my point in writing my piece today, but it seems to be everyone’s focus.

        (UNDERSTATEMENT OF THE YEAR)

        I think the focus should be more on that fact that while it’s difficult, Marco spent years building his audience.

        It doesn’t matter what he makes it how he charges. If the audience be earned is willing to pay for it, awesome.

        She tries, oh lord, she tries:

        To assert that he isn’t doing anything any other dev couldn’t, is wrong. It’s successful because it’s Marco.

        But no, HE KNOWS HER POINT BETTER THAN SHE DOES:

        No, it’s successful because he busted his ass to make it so. It’s like any other business. He grew it.

        Christ. This is like a field of strawmen. Stupid ones. Very stupid ones.

        One guy tries to blame it all on Apple, another in a string of Wha??? moments:

        the appropriate context is Apple’s App Store policies. Other devs aren’t Marco’s responsibility

        Seriously? Dude, are you even trying to talk about what Samantha actually wrote? At this point, Samantha is clearly mystified at the entire thing:

        Why has the conversation suddenly turned to focus on nothing more than ATP sponsorship income?

        Because it’s a nit they can pick and allows them to ignore everything you wrote. That’s the only reason.

        One guy is “confused”:

        I see. He does have clout, so are you saying he’s too modest in how he sees himself as a dev?

        Yes. He can’t be equated to the vast majority of other developers. Like calling Gruber, “just another blogger”.

        Alright, that’s fair. I was just confused by the $ and fame angle at first.

        Samantha’s point centers on the benefits Marco gains via his fame and background. HOW DO YOU NOT MENTION THAT? HOW IS THAT CONFUSING?

        People of course are telling her it’s her fault for mentioning a salient fact at all:

        Why has the conversation suddenly turned to focus on nothing more than ATP sponsorship income?

        Maybe because you went there with your article?

        As a way of rationalizing his ability to gamble with the potential for Overcast to generate income…not the norm at all.

        Of course, had she not brought up those important points, she’d have been bagged on for “not providing proof”. Lose some, lose more. By now, she’s had enough and she just deletes all mention of it. Understandable, but sad she was bullied into doing that.

        Yes, bullied. That’s all this is. Bullying. She didn’t lie, cheat, or exaagerate. If her numbers were wrong, they weren’t wrong in a way she had any ability to do anything about. But there’s blood in the water, and the comments and attacks get worse:

        Because you decided to start a conversation about someone else’s personal shit. You started this war.

        War. THIS. IS. WAR.

        This is a bunch of nerds attacking someone for reasoned, calm, polite criticism of their friend/idol. Samantha is politely pushing back a bit:

        That doesn’t explain why every other part of my article is being pushed aside.

        She’s right. This is all nonsense. This is people ignoring her article completely, just looking for things to attack so it can be dismissed. It’s tribalism at its purest.

        Then some of the other annointed get into it, including Jason Snell in one of the most spectactular displays of “I have special knowledge you can’t be expected to have, therefore you are totally off base and wrong, even though there’s no way for you to know this” I’ve seen in a while. Jason:

        You should never use an ad rate card to estimate ad revenue from any media product ever.

        I learned this when I started working for a magazine — rate cards are mostly fiction, like prices on new cars

        How…exactly…in the name of whatever deity Jason may believe in…is Samantha or anyone not “in the biz” supposed to know this. Also, what exactly does a magazine on paper like Macworld have to do with sponsorships for a podcast? I have done podcasts that were sponsored, and I can retaliate with “we charged what the rate card said we did. Checkmate Elitests!”

        Samantha basically abases herself at his feet:

        I understand my mistake, and it’s unfortunate that it has completely diluted the point of my article.

        I think she should have told him where and how to stuff that nonsense, but she’s a nicer person than I am. Also, it’s appropriate that Jason’s twitter avatar has its nose in the air. This is some rank snobbery. It’s disgusting and if anyone pulled that on him, Jason would be very upset. But hey, one cannot criticize The Marco without getting pushback. By “pushback”, I mean “an unrelenting fecal flood”.

        Her only mistake was criticizing one of the Kool Kids. Folks, if you criticize anyone in The Deck Clique, or their friends, expect the same thing, regardless of tone or point.

        Another App Dev, seemingly unable to parse Samantha’s words, needs more explanation:

        so just looking over your mentions, I’m curious what exactly was your main point? Ignoring the podcast income bits.

        Oh wait, he didn’t even read the article. Good on you, Dev Guy, good. on. you. Still, she plays nice with someone who didn’t even read her article:

        That a typical unknown developer can’t depend on patronage to generate revenue, and charging for apps will become a negative.

        Marco comes back of course, and now basically accuses her of lying about other devs talking to her and supporting her point:

        How many actual developers did you hear from, really? Funny how almost nobody wants to give a (real) name on these accusations.

        Really? You’re going to do that? “There’s no name, so I don’t think it’s a real person.” Just…what’s the Joe Welch quote from the McCarthy hearings?

        Let us not assassinate this lad further, Senator. You’ve done enough. Have you no sense of decency, sir? At long last, have you left no sense of decency?

        That is what this is at this point: character assasination because she said something critical of A Popular Person. It’s disgusting. Depressing and disgusting. No one, none of these people have seriously discussed her point, heck, it looks like they barely bothered to read it, if they did at all.

        Marco starts getting really petty with her (no big shock) and Samantha finally starts pushing back:

        Glad to see you be the bigger person and ignore the mindset of so many developers not relating to you, good for you!

        That of course, is what caused Marco to question the validity, if not the existence of her sources. (Funny how anonymous sources are totes okay when they convenience Marco et al, and work for oh, Apple, but when they are inconvenient? Ha! PROVIDE ME PROOF YOU INTEMPERATE WOMAN!)

        Make no mistake, there’s some sexist shit going on here. Every tweet I’ve quoted was authored by a guy.

        Of course, Marco has to play the “I’ve been around longer than you” card with this bon mot:

        Yup, before you existed!

        Really dude? I mean, I’m sorry about the penis, but really?

        Mind you, when the criticism isn’t just bizarrely stupid, Samantha reacts the way Marco and his ilk claim they would to (if they ever got any valid criticism. Which clearly is impossible):

        Not to get into the middle of this, but “income” is not the term you’re looking for. “Revenue” is.

        lol. Noted.

        And I wasn’t intending to be a dick, just a lot of people hear/say “income” when they intend “revenue”, and then discussion …

        … gets derailed by a jedi handwave of “Expenses”. But outside of charitable donation, it is all directly related.

        haha. Thank you for the clarification.

        Note to Marco and the other…whatever they are…that is how one reacts to that kind of criticism. With a bit of humor and self-deprecation. You should try it sometime. For real, not just in your heads or conversations in Irish Pubs in S.F.

        But now, the door has been cracked, and the cheap shots come out:

        @testflight_app: Don’t worry guys, we process @marcoarment’s apps in direct proportion to his megabucks earnings. #fairelephant

        (Note: testflight_app is a parody account. Please do not mess with the actual testflight folks. They are still cool.)

        Or this…conversation:

        Image for post

        Good job guys. Good job. Defend the tribe. Attack the other. Frederico attempts to recover from his stunning display of demeaning douchery: ‏@viticci: @s_bielefeld I don’t know if it’s an Italian thing, but counting other people’s money is especially weird for me. IMO, bad move in the post.

        Samantha is clearly sick of his crap: ‏@s_bielefeld: @viticci That’s what I’m referring to, the mistake of ever having mentioned it. So, now, Marco can ignore the bigger issue and go on living.

        Good for her. There’s being patient and being roadkill.

        Samantha does put the call out for her sources to maybe let her use their names:

        From all of you I heard from earlier, anyone care to go on record?

        My good friend, The Angry Drunk points out the obvious problem:

        Nobody’s going to go on record when they count on Marco’s friends for their PR.

        This is true. Again, the sites that are Friends of Marco:

        Daring Fireball

        The Loop

        SixColors

        iMore

        MacStories

        A few others, but I want this post to end one day.

        You piss that crew off, and given how petty rather a few of them have demonstrated they are, good luck on getting any kind of notice from them.

        Of course, the idea this could happen is just craycray:

        @KevinColeman .@Angry_Drunk @s_bielefeld @marcoarment Wow, you guys are veering right into crazy conspiracy theory territory. #JetFuelCantMeltSteelBeams

        Yeah. Because a mature person like Marco would never do anything like that.

        Of course, the real point on this is starting to happen:

        you’re getting a lot of heat now but happy you are writing things that stir up the community. Hope you continue to be a voice!

        I doubt I will.

        See, they’ve done their job. Mess with the bull, you get the horns. Maybe you should find another thing to write about, this isn’t a good place for you. Great job y’all.

        Some people aren’t even pretending. They’re just in full strawman mode:

        @timkeller: Unfair to begrudge a person for leveraging past success, especially when that success is earned. No ‘luck’ involved.

        @s_bielefeld: @timkeller I plainly stated that I don’t hold his doing this against him. Way to twist words.

        I think she’s earned her anger at this point.

        Don’t worry, Marco knows what the real problem is: most devs just suck —

        Image for post

        I have a saying that applies in this case: don’t place your head so far up your nethers that you go full Klein Bottle. Marco has gone full Klein Bottle. (To be correct, he went FKB some years ago.)

        There are some bright spots. My favorite is when Building Twenty points out the real elephant in the room:

        @BuildingTwenty: Both @s_bielefeld & I wrote similar critiques of @marcoarment’s pricing model yet the Internet pilloried only the woman. Who’d have guessed?

        Yup.

        Another bright spot are these comments from Ian Betteridge, who has been doing this even longer than Marco:

        You know, any writer who has never made a single factual error in a piece hasn’t ever written anything worth reading.

        I learned my job with the support of people who helped me. Had I suffered an Internet pile on for every error I wouldn’t have bothered.

        To which Samantha understandably replies:

        and it’s honestly something I’m contemplating right now, whether to continue…

        Gee, I can’t imagine why. Why with comments like this from Chris Breen that completely misrepresent Samantha’s point, (who until today, I would have absolutely defended as being better than this, something I am genuinely saddened to be wrong about), why wouldn’t she want to continue doing this?

        If I have this right, some people are outraged that a creator has decided to give away his work.

        No Chris, you don’t have this right. But hey, who has time to find out the real issue and read an article. I’m sure your friends told you everything you need to know.

        Noted Feminist Glenn Fleishman gets a piece of the action too:

        Image for post

        I’m not actually surprised here. I watched Fleishman berate a friend of mine who has been an engineer for…heck, waaaaay too long on major software products in the most condescending way because she tried to point out that as a very technical woman, “The Magazine” literally had nothing to say to her and maybe he should fix that. “Impertinent” was I believe what he called her, but I may have the specific word wrong. Not the attitude mind you. Great Feminists like Glenn do not like uppity women criticizing Great Feminists who are their Great Allies.

        Great Feminists are often tools.

        Luckily, I hope, the people who get Samantha’s point also started chiming in (and you get 100% of the women commenting here that I’ve seen):

        I don’t think he’s wrong for doing it, he just discusses it as if the market’s a level playing field — it isn’t

        This is a great article with lots of great points about the sustainability of iOS development. Thank you for publishing it.

        Regardless of the numbers and your view of MA, fair points here about confirmation bias in app marketing feasibility http://samanthabielefeld.com/the-elephant-in-the-room …

        thank you for posting this, it covers a lot of things people don’t like to talk about.

        I’m sure you have caught untold amounts of flak over posting this because Marco is blind to his privilege as a developer.

        Catching up on the debate, and agreeing with Harry’s remark. (Enjoyed your article, Samantha, and ‘got’ your point.)

        I would like to say I’m surprised at the reaction to Samantha’s article, but I’m not. In spite of his loud declarations of support for The Big Lie, Marco Arment is as bad at any form of criticism that he hasn’t already approved as a very insecure tween. An example from 2011: http://www.businessinsider.com/marco-arment-2011-9

        Marco is great with criticism as long as it never actually criticizes him. If it does, be prepared a flood of petty, petulant whining that a room full of bored preschoolers on a hot day would be hard-pressed to match.

        Today has been…well, it sucks. It sucks because someone doing what all the Arments of the world claim to want was naive enough to believe what they were told, and found out the hard way just how big a lie The Big Lie is, and how vicious people are when you’re silly enough to believe anything they say about criticism.

        And note again, every single condescending crack, misrepresentation, and strawman had an exclusively male source. Most of them have, at one point or another, loudly trumpted themselves as Feminist Allies, as a friend to women struggling with the sexism and misogyny in tech. Congratulations y’all on being just as bad as the people you claim to oppose.

        Samantha has handled this better than anyone else could have. My respect for her as a person and a writer is off the charts. If she choses to walk away from blogging in the Apple space, believe me I understand. As bad as today was for her, I’ve seen worse. Much worse.

        But I hope she doesn’t. I hope she stays, because she is Doing This Right, and in a corner of the internet that has become naught but an endless circle jerk, a cliquish collection, a churlish, childish cohort interested not in writing or the truth, but in making sure The Right People are elevated, and The Others put down, she is someone worth reading and listening to. The number people who owe her apologies goes around the block, and I don’t think she’ll ever see a one. I’m sure as heck not apologizing for them, I’ll not make their lives easier in the least.

        All of you, all. of. you…Marco, Breen, Snell, Vittici, had a chance to live by your words. You were faced with reasoned, polite, respectful criticism and instead of what you should have done, you all dropped trou and sprayed an epic diarrheal discharge all over someone who had done nothing to deserve it. Me, I earned most of my aggro, Samantha did not earn any of the idiocy I’ve seen today. I hope you’re all proud of yourselves. Someone should be, it won’t be me. Ever.

        So I hope she stays, but if she goes, I understand. For what it’s worth, I don’t think she’s wrong either way.

        Discover Medium

        Welcome to a place where words matter. On Medium, smart voices and original ideas take center stage - with no ads in sight. Watch

        Make Medium yours

        Follow all the topics you care about, and we’ll deliver the best stories for you to your homepage and inbox. Explore

        Become a member

        Get unlimited access to the best stories on Medium — and support writers while you’re at it. Just $5/month. Upgrade

        About

        Help

        Legal

        Get the Medium app

        A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
        A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
        ================================================ FILE: test/test-pages/mercurial/expected-metadata.json ================================================ { "title": "Evolve: Shared Mutable History — evolve extension for Mercurial", "byline": null, "dir": null, "excerpt": "Contents", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/mercurial/expected.html ================================================

        Once you have mastered the art of mutable history in a single repository (see the user guide), you can move up to the next level: shared mutable history. evolve lets you push and pull draft changesets between repositories along with their obsolescence markers. This opens up a number of interesting possibilities.

        The simplest scenario is a single developer working across two computers. Say you’re working on code that must be tested on a remote test server, probably in a rack somewhere, only accessible by SSH, and running an “enterprise-grade” (out-of-date) OS. But you probably prefer to write code locally: everything is setup the way you like it, and you can use your preferred editor, IDE, merge/diff tools, etc.

        Traditionally, your options are limited: either

        • (ab)use your source control system by committing half-working code in order to get it onto the remote test server, or
        • go behind source control’s back by using rsync (or similar) to transfer your code back-and-forth until it is ready to commit

        The former is less bad with distributed version control systems like Mercurial, but it’s still far from ideal. (One important version control “best practice” is that every commit should make things just a little bit better, i.e. you should never commit code that is worse than what came before.) The latter, avoiding version control entirely, means that you’re walking a tightrope without a safety net. One accidental rsync in the wrong direction could destroy hours of work.

        Using Mercurial with evolve to share mutable history solves these problems. As with single-repository evolve, you can commit whenever the code is demonstrably better, even if all the tests aren’t passing yet—just hg amend when they are. And you can transfer those half-baked changesets between repositories to try things out on your test server before anything is carved in stone.

        A less common scenario is multiple developers sharing mutable history, typically for code review. We’ll cover this scenario later. First, we will cover single-user sharing.

        Sharing with a single developer

        Publishing and non-publishing repositories

        The key to shared mutable history is to keep your changesets in draft phase as you pass them around. Recall that by default, hg push promotes changesets from draft to public, and public changesets are immutable. You can change this behaviour by reconfiguring the remote repository so that it is non-publishing. (Short version: set phases.publish to false. Long version follows.)

        Setting up

        We’ll work through an example with three local repositories, although in the real world they’d most likely be on three different computers. First, the public repository is where tested, polished changesets live, and it is where you synchronize with the rest of your team. 

        $ hg init public

        We’ll need two clones where work gets done, test-repo and dev-repo: 

        $ hg clone public test-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone test-repo dev-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved

        dev-repo is your local machine, with GUI merge tools and IDEs and everything configured just the way you like it. test-repo is the test server in a rack somewhere behind SSH. So for the most part, we’ll develop in dev-repo, push to test-repo, test and polish there, and push to public.

        The key to shared mutable history is to make the target repository, in this case test-repo, non-publishing. And, of course, we have to enable the evolve extension in both test-repo and dev-repo.

        First, edit the configuration for test-repo: 

        $ hg -R test-repo config --edit --local

        and add 

        [phases]
publish = false

[extensions]
evolve =

        Then edit the configuration for dev-repo: 

        $ hg -R dev-repo config --edit --local

        and add 

        [extensions]
evolve =

        Keep in mind that in real life, these repositories would probably be on separate computers, so you’d have to login to each one to configure each repository.

        To start things off, let’s make one public, immutable changeset: 

        $ cd test-repo
$ echo 'my new project' > file1
$ hg add file1
$ hg commit -m 'create new project'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files

        and pull that into the development repository: 

        $ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        Example 2: Amend again, locally

        This process can repeat. Perhaps you figure out a more elegant fix to the bug, and want to mutate history so nobody ever knows you had a less-than-perfect idea. We’ll implement it locally in dev-repo and push to test-repo: 

        $ echo 'Fix, fix, and fix.' > file1
$ hg amend
$ hg push

        This time around, the temporary amend commit is in dev-repo, and it is not transferred to test-repo—the same as before, just in the opposite direction. Figure 4 shows the two repositories after amending in dev-repo and pushing to test-repo.

        [figure SG04: each repo has one temporary amend commit, but they’re different in each one]

        Let’s hop over to test-repo to test the more elegant fix: 

        $ cd ../test-repo
$ hg update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        This time, all the tests pass, so no further amending is required. This bug fix is finished, so we push it to the public repository: 

        $ hg push
[...]
added 1 changesets with 1 changes to 1 files

        Note that only one changeset—the final version, after two amendments—was actually pushed. Again, Mercurial doesn’t transfer hidden changesets on push and pull.

        So the picture in public is much simpler than in either dev-repo or test-repo. Neither of our missteps nor our amendments are publicly visible, just the final, beautifully polished changeset:

        [figure SG05: public repo with rev 0:0dc9, 1:de61, both public]

        There is one important step left to do. Because we pushed from test-repo to public, the pushed changeset is in public phase in those two repositories. But dev-repo has been out-of-the-loop; changeset de61 is still draft there. If we’re not careful, we might mutate history in dev-repo, obsoleting a changeset that is already public. Let’s avoid that situation for now by pushing up to dev-repo: 

        $ hg push ../dev-repo
pushing to ../dev-repo
searching for changes
no changes found

        Even though no changesets were pushed, Mercurial still pushed obsolescence markers and phase changes to dev-repo.

        A final note: since this fix is now public, it is immutable. It’s no longer possible to amend it: 

        $ hg amend -m 'fix bug 37'
abort: cannot amend public changesets

        This is, after all, the whole point of Mercurial’s phases: to prevent rewriting history that has already been published.

        Sharing with multiple developers: code review

        Now that you know how to share your own mutable history across multiple computers, you might be wondering if it makes sense to share mutable history with others. It does, but you have to be careful, stay alert, and communicate with your peers.

        Code review is a good use case for sharing mutable history across multiple developers: Alice commits a draft changeset, submits it for review, and amends her changeset until her reviewer is satisfied. Meanwhile, Bob is also committing draft changesets for review, amending until his reviewer is satisfied. Once a particular changeset passes review, the respective author (Alice or Bob) pushes it to the public (publishing) repository.

        Incidentally, the reviewers here can be anyone: maybe Bob and Alice review each other’s work; maybe the same third party reviews both; or maybe they pick different experts to review their work on different parts of a large codebase. Similarly, it doesn’t matter if reviews are conducted in person, by email, or by carrier pigeon. Code review is outside of the scope of Mercurial, so all we’re looking at here is the mechanics of committing, amending, pushing, and pulling.

        Setting up

        To demonstrate, let’s start with the public repository as we left it in the last example, with two immutable changesets (figure 5 above). We’ll clone a review repository from it, and then Alice and Bob will both clone from review. 

        $ hg clone public review
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review alice
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review bob
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        We need to configure Alice’s and Bob’s working repositories to enable evolve. First, edit Alice’s configuration with 

        $ hg -R alice config --edit --local

        and add 

        [extensions]
evolve =

        Then edit Bob’s repository configuration: 

        $ hg -R bob config --edit --local

        and add the same text.

        Example 3: Alice commits and amends a draft fix

        We’ll follow Alice working on a bug fix. We’re going to use bookmarks to make it easier to understand multiple branch heads in the review repository, so Alice starts off by creating a bookmark and committing her first attempt at a fix: 

        $ hg bookmark bug15
$ echo 'fix' > file2
$ hg commit -A -u alice -m 'fix bug 15 (v1)'
adding file2

        Note the unorthodox “(v1)” in the commit message. We’re just using that to make this tutorial easier to follow; it’s not something we’d recommend in real life.

        Of course Alice wouldn’t commit unless her fix worked to her satisfaction, so it must be time to solicit a code review. She does this by pushing to the review repository: 

        $ hg push -B bug15
[...]
added 1 changesets with 1 changes to 1 files
exporting bookmark bug15

        (The use of -B is important to ensure that we only push the bookmarked head, and that the bookmark itself is pushed. See this guide to bookmarks, especially the Sharing Bookmarks section, if you’re not familiar with bookmarks.)

        Some time passes, and Alice receives her code review. As a result, Alice revises her fix and submits it for a second review: 

        $ echo 'Fix.' > file2
$ hg amend -m 'fix bug 15 (v2)'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark bug15

        Figure 6 shows the state of the review repository at this point.

        [figure SG06: rev 2:fn1e is Alice’s obsolete v1, rev 3:cbdf is her v2; both children of rev 1:de61]

        After a busy morning of bug fixing, Alice stops for lunch. Let’s see what Bob has been up to.

        Example 4: Bob implements and publishes a new feature

        Meanwhile, Bob has been working on a new feature. Like Alice, he’ll use a bookmark to track his work, and he’ll push that bookmark to the review repository, so that reviewers know which changesets to review. 

        $ cd ../bob
$ echo 'stuff' > file1
$ hg bookmark featureX
$ hg commit -u bob -m 'implement feature X (v1)'          # rev 4:1636
$ hg push -B featureX
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
exporting bookmark featureX

        When Bob receives his code review, he improves his implementation a bit, amends, and submits the resulting changeset for review: 

        $ echo 'do stuff' > file1
$ hg amend -m 'implement feature X (v2)'                  # rev 5:0eb7
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX

        Unfortunately, that still doesn’t pass muster. Bob’s reviewer insists on proper capitalization and punctuation. 

        $ echo 'Do stuff.' > file1
$ hg amend -m 'implement feature X (v3)'                  # rev 6:540b

        On the bright side, the second review said, “Go ahead and publish once you fix that.” So Bob immediately publishes his third attempt: 

        $ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files

        It’s not enough just to update public, though! Other people also use the review repository, and right now it doesn’t have Bob’s latest amendment (“v3”, revision 6:540b), nor does it know that the precursor of that changeset (“v2”, revision 5:0eb7) is obsolete. Thus, Bob pushes to review as well: 

        $ hg push ../review
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX

        Figure 7 shows the result of Bob’s work in both review and public.

        [figure SG07: review includes Alice’s draft work on bug 15, as well as Bob’s v1, v2, and v3 changes for feature X: v1 and v2 obsolete, v3 public. public contains only the final, public implementation of feature X]

        Incidentally, it’s important that Bob push to public before review. If he pushed to review first, then revision 6:540b would still be in draft phase in review, but it would be public in both Bob’s local repository and the public repository. That could lead to confusion at some point, which is easily avoided by pushing first to public.

        Example 5: Alice integrates and publishes

        Finally, Alice gets back from lunch and sees that the carrier pigeon with her second review has arrived (or maybe it’s in her email inbox). Alice’s reviewer approved her amended changeset, so she pushes it to public: 

        $ hg push ../public
[...]
remote has heads on branch 'default' that are not known locally: 540ba8f317e6
abort: push creates new remote head cbdfbd5a5db2!
(pull and merge or see "hg help push" for details about pushing new heads)

        Oops! Bob has won the race to push first to public. So Alice needs to integrate with Bob: let’s pull his changeset(s) and see what the branch heads are. 

        $ hg pull ../public
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
$ hg log -G -q -r 'head()' --template '{rev}:{node|short}  ({author})\n'
o  5:540ba8f317e6  (bob)
|
| @  4:cbdfbd5a5db2  (alice)
|/

        We’ll assume Alice and Bob are perfectly comfortable with rebasing changesets. (After all, they’re already using mutable history in the form of amend.) So Alice rebases her changeset on top of Bob’s and publishes the result: 

        $ hg rebase -d 5
$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
$ hg push ../review
[...]
added 1 changesets with 0 changes to 0 files
updating bookmark bug15

        The result, in both review and public repositories, is shown in figure 8.

        [figure SG08: review shows v1 and v2 of Alice’s fix, then v1, v2, v3 of Bob’s feature, finally Alice’s fix rebased onto Bob’s. public just shows the final public version of each changeset]

        Getting into trouble with shared mutable history

        Mercurial with evolve is a powerful tool, and using powerful tools can have consequences. (You can cut yourself badly with a sharp knife, but every competent chef keeps several around. Ever try to chop onions with a spoon?)

        In the user guide, we saw examples of unstbale changesets, which are the most common type of troubled changeset. (Recall that a non-obsolete changeset with obsolete ancestors is an orphan.)

        Two other types of troubles can happen: divergent and bumped changesets. Both are more likely with shared mutable history, especially mutable history shared by multiple developers.

        Setting up

        For these examples, we’re going to use a slightly different workflow: as before, Alice and Bob share a public repository. But this time there is no review repository. Instead, Alice and Bob put on their cowboy hats, throw good practice to the wind, and pull directly from each other’s working repositories.

        So we throw away everything except public and reclone: 

        $ rm -rf review alice bob
$ hg clone public alice
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone public bob
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved

        Once again we have to configure their repositories: enable evolve and (since Alice and Bob will be pulling directly from each other) make their repositories non-publishing. Edit Alice’s configuration: 

        $ hg -R alice config --edit --local

        and add 

        [extensions]
rebase =
evolve =

[phases]
publish = false

        Then edit Bob’s repository configuration: 

        $ hg -R bob config --edit --local

        and add the same text.

        Example 6: Divergent changesets

        When an obsolete changeset has two successors, those successors are divergent. One way to get into such a situation is by failing to communicate with your teammates. Let’s see how that might happen.

        First, we’ll have Bob commit a bug fix that could still be improved: 

        $ cd bob
$ echo 'pretty good fix' >> file1
$ hg commit -u bob -m 'fix bug 24 (v1)'                   # rev 4:2fe6

        Since Alice and Bob are now in cowboy mode, Alice pulls Bob’s draft changeset and amends it herself. 

        $ cd ../alice
$ hg pull -u ../bob
[...]
added 1 changesets with 1 changes to 1 files
$ echo 'better fix (alice)' >> file1
$ hg amend -u alice -m 'fix bug 24 (v2 by alice)'

        But Bob has no idea that Alice just did this. (See how important good communication is?) So he implements a better fix of his own: 

        $ cd ../bob
$ echo 'better fix (bob)' >> file1
$ hg amend -u bob -m 'fix bug 24 (v2 by bob)'             # rev 6:a360

        At this point, the divergence exists, but only in theory: Bob’s original changeset, 4:2fe6, is obsolete and has two successors. But those successors are in different repositories, so the trouble is not visible to anyone yet. It will be as soon as Bob pulls from Alice’s repository (or vice-versa). 

        $ hg pull ../alice
[...]
added 1 changesets with 1 changes to 2 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
2 new divergent changesets

        Figure 9 shows the situation in Bob’s repository.

        [figure SG09: Bob’s repo with 2 heads for the 2 divergent changesets, 6:a360 and 7:e3f9; wc is at 6:a360; both are successors of obsolete 4:2fe6, hence divergence]

        Now we need to get out of trouble. As usual, the answer is to evolve history. 

        $ HGMERGE=internal:other hg evolve
merge:[6] fix bug 24 (v2 by bob)
with: [7] fix bug 24 (v2 by alice)
base: [4] fix bug 24 (v1)
0 files updated, 1 files merged, 0 files removed, 0 files unresolved

        Figure 10 shows how Bob’s repository looks now.

        [figure SG10: only one visible head, 9:5ad6, successor to hidden 6:a360 and 7:e3f9]

        We carefully dodged a merge conflict by specifying a merge tool (internal:other) that will take Alice’s changes over Bob’s. (You might wonder why Bob wouldn’t prefer his own changes by using internal:local. He’s avoiding a bug in evolve that occurs when evolving divergent changesets using internal:local.)

        # XXX this link does not work .. bug: https://bitbucket.org/marmoute/mutable-history/issue/48/

        ** STOP HERE: WORK IN PROGRESS **

        Phase-divergence: when a rewritten changeset is made public

        If Alice and Bob are collaborating on some mutable changesets, it’s possible to get into a situation where an otherwise worthwhile changeset cannot be pushed to the public repository; it is phase-divergent with another changeset that was made public first. Let’s demonstrate one way this could happen.

        It starts with Alice committing a bug fix. Right now, we don’t yet know if this bug fix is good enough to push to the public repository, but it’s good enough for Alice to commit. 

        $ cd alice
$ echo 'fix' > file2
$ hg commit -A -m 'fix bug 15'
adding file2

        Now Bob has a bad idea: he decides to pull whatever Alice is working on and tweak her bug fix to his taste: 

        $ cd ../bob
$ hg pull -u ../alice
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ echo 'Fix.' > file2
$ hg amend -A -m 'fix bug 15 (amended)'

        (Note the lack of communication between Alice and Bob. Failing to communicate with your colleagues is a good way to get into trouble. Nevertheless, evolve can usually sort things out, as we will see.)

        [figure SG06: Bob’s repo with one amendment]

        After some testing, Alice realizes her bug fix is just fine as it is: no need for further polishing and amending, this changeset is ready to publish. 

        $ cd ../alice
$ hg push
[...]
added 1 changesets with 1 changes to 1 files

        This introduces a contradiction: in Bob’s repository, changeset 2:e011 (his copy of Alice’s fix) is obsolete, since Bob amended it. But in Alice’s repository (and the public repository), that changeset is public: it is immutable, carved in stone for all eternity. No changeset can be both obsolete and public, so Bob is in for a surprise the next time he pulls from public: 

        $ cd ../bob
$ hg pull -q -u
1 new phase-divergent changesets

        Figure 7 shows what just happened to Bob’s repository: changeset 2:e011 is now public, so it can’t be obsolete. When that changeset was obsolete, it made perfect sense for it to have a successor, namely Bob’s amendment of Alice’s fix (changeset 4:fe88). But it’s illogical for a public changeset to have a successor, so 4:fe88 is troubled: it has become bumped.

        [figure SG07: 2:e011 now public not obsolete, 4:fe88 now bumped]

        As usual when there’s trouble in your repository, the solution is to evolve it: 

        $ hg evolve --all

        Figure 8 illustrates Bob’s repository after evolving away the bumped changeset. Ignoring the obsolete changesets, Bob now has a nice, clean, simple history. His amendment of Alice’s bug fix lives on, as changeset 5:227d—albeit with a software-generated commit message. (Bob should probably amend that changeset to improve the commit message.) But the important thing is that his repository no longer has any troubled changesets, thanks to evolve.

        [figure SG08: 5:227d is new, formerly bumped changeset 4:fe88 now hidden]

        Conclusion

        Mutable history is a powerful tool. Like a sharp knife, an experienced user can do wonderful things with it, much more wonderful than with a dull knife (never mind a rusty spoon). At the same time, an inattentive or careless user can do harm to himself or others. Mercurial with evolve goes to great lengths to limit the harm you can do by trying to handle all possible types of “troubled” changesets. Nevertheless, having a first-aid kit nearby does not mean you should stop being careful with sharp knives.

        Mutable history shared across multiple repositories by a single developer is a natural extension of this model. Once you are used to using a single sharp knife on its own, it’s pretty straightforward to chop onions and mushrooms using the same knife, or to alternate between two chopping boards with different knives.

        Mutable history shared by multiple developers is a scary place to go. Imagine a professional kitchen full of expert chefs tossing their favourite knives back and forth, with the occasional axe or chainsaw thrown in to spice things up. If you’re confident that you and your colleagues can do it without losing a limb, go for it. But be sure to practice a lot first before you rely on it!

        ================================================ FILE: test/test-pages/mercurial/source.html ================================================ Evolve: Shared Mutable History — evolve extension for Mercurial

        evolve extension for Mercurial

        Evolve: Shared Mutable History

        «  Evolve: User Guide   ::   Contents   ::   Evolve: Concepts  »

        Evolve: Shared Mutable History

        Once you have mastered the art of mutable history in a single repository (see the user guide), you can move up to the next level: shared mutable history. evolve lets you push and pull draft changesets between repositories along with their obsolescence markers. This opens up a number of interesting possibilities.

        The simplest scenario is a single developer working across two computers. Say you’re working on code that must be tested on a remote test server, probably in a rack somewhere, only accessible by SSH, and running an “enterprise-grade” (out-of-date) OS. But you probably prefer to write code locally: everything is setup the way you like it, and you can use your preferred editor, IDE, merge/diff tools, etc.

        Traditionally, your options are limited: either

        • (ab)use your source control system by committing half-working code in order to get it onto the remote test server, or
        • go behind source control’s back by using rsync (or similar) to transfer your code back-and-forth until it is ready to commit

        The former is less bad with distributed version control systems like Mercurial, but it’s still far from ideal. (One important version control “best practice” is that every commit should make things just a little bit better, i.e. you should never commit code that is worse than what came before.) The latter, avoiding version control entirely, means that you’re walking a tightrope without a safety net. One accidental rsync in the wrong direction could destroy hours of work.

        Using Mercurial with evolve to share mutable history solves these problems. As with single-repository evolve, you can commit whenever the code is demonstrably better, even if all the tests aren’t passing yet—just hg amend when they are. And you can transfer those half-baked changesets between repositories to try things out on your test server before anything is carved in stone.

        A less common scenario is multiple developers sharing mutable history, typically for code review. We’ll cover this scenario later. First, we will cover single-user sharing.

        Sharing with a single developer

        Publishing and non-publishing repositories

        The key to shared mutable history is to keep your changesets in draft phase as you pass them around. Recall that by default, hg push promotes changesets from draft to public, and public changesets are immutable. You can change this behaviour by reconfiguring the remote repository so that it is non-publishing. (Short version: set phases.publish to false. Long version follows.)

        Setting up

        We’ll work through an example with three local repositories, although in the real world they’d most likely be on three different computers. First, the public repository is where tested, polished changesets live, and it is where you synchronize with the rest of your team. 

        $ hg init public

        We’ll need two clones where work gets done, test-repo and dev-repo: 

        $ hg clone public test-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone test-repo dev-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved

        dev-repo is your local machine, with GUI merge tools and IDEs and everything configured just the way you like it. test-repo is the test server in a rack somewhere behind SSH. So for the most part, we’ll develop in dev-repo, push to test-repo, test and polish there, and push to public.

        The key to shared mutable history is to make the target repository, in this case test-repo, non-publishing. And, of course, we have to enable the evolve extension in both test-repo and dev-repo.

        First, edit the configuration for test-repo: 

        $ hg -R test-repo config --edit --local

        and add 

        [phases]
publish = false

[extensions]
evolve =

        Then edit the configuration for dev-repo: 

        $ hg -R dev-repo config --edit --local

        and add 

        [extensions]
evolve =

        Keep in mind that in real life, these repositories would probably be on separate computers, so you’d have to login to each one to configure each repository.

        To start things off, let’s make one public, immutable changeset: 

        $ cd test-repo
$ echo 'my new project' > file1
$ hg add file1
$ hg commit -m 'create new project'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files

        and pull that into the development repository: 

        $ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        Example 1: Amend a shared changeset

        Everything you learned in the user guide applies to work done in dev-repo. You can commit, amend, uncommit, evolve, and so forth just as before.

        Things get different when you push changesets to test-repo. Or rather, things stay the same, which is different: because we configured test-repo to be non-publishing, draft changesets stay draft when we push them to test-repo. Importantly, they’re also draft (mutable) in test-repo.

        Let’s commit a preliminary change and push it to test-repo for testing. 

        $ echo 'fix fix fix' > file1
$ hg commit -m 'prelim change'
$ hg push ../test-repo

        At this point, dev-repo and test-repo have the same changesets in the same phases:

        [figure SG01: rev 0:0dc9 public, rev 1:f649 draft, same on both repos]

        (You may notice a change in notation from the user guide: now changesets are labelled with their revision number and the first four digits of the 40-digit hexadecimal changeset ID. Mercurial revision numbers are never stable when working across repositories, especially when obsolescence is involved. We’ll see why shortly.)

        Now let’s switch to test-repo to test our change: 

        $ cd ../test-repo
$ hg update

        Don’t forget to hg update! Pushing only adds changesets to a remote repository; it does not update the working directory (unless you have a hook that updates for you).

        Now let’s imagine the tests failed because we didn’t use proper punctuation and capitalization (oops). Let’s amend our preliminary fix (and fix the lame commit message while we’re at it): 

        $ echo 'Fix fix fix.' > file1
$ hg amend -m 'fix bug 37'

        Now we’re in a funny intermediate state (figure 2): revision 1:f649 is obsolete in test-repo, having been replaced by revision 3:60ff (revision 2:2a03 is another one of those temporary amend commits that we saw in the user guide)—but dev-repo knows nothing of these recent developments.

        [figure SG02: test-repo has rev 0:0dc9 public, rev 1:f649, 2:2a03 obsolete, rev 3:60ff draft; dev-repo same as in SG01]

        Let’s resynchronize: 

        $ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        As seen in figure 3, this transfers the new changeset and the obsolescence marker for revision 1. However, it does not transfer the temporary amend commit, because it is hidden. Push and pull transfer obsolescence markers between repositories, but they do not transfer hidden changesets.

        [figure SG03: dev-repo grows new rev 2:60ff, marks 1:f649 obsolete]

        Because of this deliberately incomplete synchronization, revision numbers in test-repo and dev-repo are no longer consistent. We must use changeset IDs.

        Example 2: Amend again, locally

        This process can repeat. Perhaps you figure out a more elegant fix to the bug, and want to mutate history so nobody ever knows you had a less-than-perfect idea. We’ll implement it locally in dev-repo and push to test-repo: 

        $ echo 'Fix, fix, and fix.' > file1
$ hg amend
$ hg push

        This time around, the temporary amend commit is in dev-repo, and it is not transferred to test-repo—the same as before, just in the opposite direction. Figure 4 shows the two repositories after amending in dev-repo and pushing to test-repo.

        [figure SG04: each repo has one temporary amend commit, but they’re different in each one]

        Let’s hop over to test-repo to test the more elegant fix: 

        $ cd ../test-repo
$ hg update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        This time, all the tests pass, so no further amending is required. This bug fix is finished, so we push it to the public repository: 

        $ hg push
[...]
added 1 changesets with 1 changes to 1 files

        Note that only one changeset—the final version, after two amendments—was actually pushed. Again, Mercurial doesn’t transfer hidden changesets on push and pull.

        So the picture in public is much simpler than in either dev-repo or test-repo. Neither of our missteps nor our amendments are publicly visible, just the final, beautifully polished changeset:

        [figure SG05: public repo with rev 0:0dc9, 1:de61, both public]

        There is one important step left to do. Because we pushed from test-repo to public, the pushed changeset is in public phase in those two repositories. But dev-repo has been out-of-the-loop; changeset de61 is still draft there. If we’re not careful, we might mutate history in dev-repo, obsoleting a changeset that is already public. Let’s avoid that situation for now by pushing up to dev-repo: 

        $ hg push ../dev-repo
pushing to ../dev-repo
searching for changes
no changes found

        Even though no changesets were pushed, Mercurial still pushed obsolescence markers and phase changes to dev-repo.

        A final note: since this fix is now public, it is immutable. It’s no longer possible to amend it: 

        $ hg amend -m 'fix bug 37'
abort: cannot amend public changesets

        This is, after all, the whole point of Mercurial’s phases: to prevent rewriting history that has already been published.

        Sharing with multiple developers: code review

        Now that you know how to share your own mutable history across multiple computers, you might be wondering if it makes sense to share mutable history with others. It does, but you have to be careful, stay alert, and communicate with your peers.

        Code review is a good use case for sharing mutable history across multiple developers: Alice commits a draft changeset, submits it for review, and amends her changeset until her reviewer is satisfied. Meanwhile, Bob is also committing draft changesets for review, amending until his reviewer is satisfied. Once a particular changeset passes review, the respective author (Alice or Bob) pushes it to the public (publishing) repository.

        Incidentally, the reviewers here can be anyone: maybe Bob and Alice review each other’s work; maybe the same third party reviews both; or maybe they pick different experts to review their work on different parts of a large codebase. Similarly, it doesn’t matter if reviews are conducted in person, by email, or by carrier pigeon. Code review is outside of the scope of Mercurial, so all we’re looking at here is the mechanics of committing, amending, pushing, and pulling.

        Setting up

        To demonstrate, let’s start with the public repository as we left it in the last example, with two immutable changesets (figure 5 above). We’ll clone a review repository from it, and then Alice and Bob will both clone from review. 

        $ hg clone public review
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review alice
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review bob
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

        We need to configure Alice’s and Bob’s working repositories to enable evolve. First, edit Alice’s configuration with 

        $ hg -R alice config --edit --local

        and add 

        [extensions]
evolve =

        Then edit Bob’s repository configuration: 

        $ hg -R bob config --edit --local

        and add the same text.

        Example 3: Alice commits and amends a draft fix

        We’ll follow Alice working on a bug fix. We’re going to use bookmarks to make it easier to understand multiple branch heads in the review repository, so Alice starts off by creating a bookmark and committing her first attempt at a fix: 

        $ hg bookmark bug15
$ echo 'fix' > file2
$ hg commit -A -u alice -m 'fix bug 15 (v1)'
adding file2

        Note the unorthodox “(v1)” in the commit message. We’re just using that to make this tutorial easier to follow; it’s not something we’d recommend in real life.

        Of course Alice wouldn’t commit unless her fix worked to her satisfaction, so it must be time to solicit a code review. She does this by pushing to the review repository: 

        $ hg push -B bug15
[...]
added 1 changesets with 1 changes to 1 files
exporting bookmark bug15

        (The use of -B is important to ensure that we only push the bookmarked head, and that the bookmark itself is pushed. See this guide to bookmarks, especially the Sharing Bookmarks section, if you’re not familiar with bookmarks.)

        Some time passes, and Alice receives her code review. As a result, Alice revises her fix and submits it for a second review: 

        $ echo 'Fix.' > file2
$ hg amend -m 'fix bug 15 (v2)'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark bug15

        Figure 6 shows the state of the review repository at this point.

        [figure SG06: rev 2:fn1e is Alice’s obsolete v1, rev 3:cbdf is her v2; both children of rev 1:de61]

        After a busy morning of bug fixing, Alice stops for lunch. Let’s see what Bob has been up to.

        Example 4: Bob implements and publishes a new feature

        Meanwhile, Bob has been working on a new feature. Like Alice, he’ll use a bookmark to track his work, and he’ll push that bookmark to the review repository, so that reviewers know which changesets to review. 

        $ cd ../bob
$ echo 'stuff' > file1
$ hg bookmark featureX
$ hg commit -u bob -m 'implement feature X (v1)'          # rev 4:1636
$ hg push -B featureX
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
exporting bookmark featureX

        When Bob receives his code review, he improves his implementation a bit, amends, and submits the resulting changeset for review: 

        $ echo 'do stuff' > file1
$ hg amend -m 'implement feature X (v2)'                  # rev 5:0eb7
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX

        Unfortunately, that still doesn’t pass muster. Bob’s reviewer insists on proper capitalization and punctuation. 

        $ echo 'Do stuff.' > file1
$ hg amend -m 'implement feature X (v3)'                  # rev 6:540b

        On the bright side, the second review said, “Go ahead and publish once you fix that.” So Bob immediately publishes his third attempt: 

        $ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files

        It’s not enough just to update public, though! Other people also use the review repository, and right now it doesn’t have Bob’s latest amendment (“v3”, revision 6:540b), nor does it know that the precursor of that changeset (“v2”, revision 5:0eb7) is obsolete. Thus, Bob pushes to review as well: 

        $ hg push ../review
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX

        Figure 7 shows the result of Bob’s work in both review and public.

        [figure SG07: review includes Alice’s draft work on bug 15, as well as Bob’s v1, v2, and v3 changes for feature X: v1 and v2 obsolete, v3 public. public contains only the final, public implementation of feature X]

        Incidentally, it’s important that Bob push to public before review. If he pushed to review first, then revision 6:540b would still be in draft phase in review, but it would be public in both Bob’s local repository and the public repository. That could lead to confusion at some point, which is easily avoided by pushing first to public.

        Example 5: Alice integrates and publishes

        Finally, Alice gets back from lunch and sees that the carrier pigeon with her second review has arrived (or maybe it’s in her email inbox). Alice’s reviewer approved her amended changeset, so she pushes it to public: 

        $ hg push ../public
[...]
remote has heads on branch 'default' that are not known locally: 540ba8f317e6
abort: push creates new remote head cbdfbd5a5db2!
(pull and merge or see "hg help push" for details about pushing new heads)

        Oops! Bob has won the race to push first to public. So Alice needs to integrate with Bob: let’s pull his changeset(s) and see what the branch heads are. 

        $ hg pull ../public
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
$ hg log -G -q -r 'head()' --template '{rev}:{node|short}  ({author})\n'
o  5:540ba8f317e6  (bob)
|
| @  4:cbdfbd5a5db2  (alice)
|/

        We’ll assume Alice and Bob are perfectly comfortable with rebasing changesets. (After all, they’re already using mutable history in the form of amend.) So Alice rebases her changeset on top of Bob’s and publishes the result: 

        $ hg rebase -d 5
$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
$ hg push ../review
[...]
added 1 changesets with 0 changes to 0 files
updating bookmark bug15

        The result, in both review and public repositories, is shown in figure 8.

        [figure SG08: review shows v1 and v2 of Alice’s fix, then v1, v2, v3 of Bob’s feature, finally Alice’s fix rebased onto Bob’s. public just shows the final public version of each changeset]

        Getting into trouble with shared mutable history

        Mercurial with evolve is a powerful tool, and using powerful tools can have consequences. (You can cut yourself badly with a sharp knife, but every competent chef keeps several around. Ever try to chop onions with a spoon?)

        In the user guide, we saw examples of unstbale changesets, which are the most common type of troubled changeset. (Recall that a non-obsolete changeset with obsolete ancestors is an orphan.)

        Two other types of troubles can happen: divergent and bumped changesets. Both are more likely with shared mutable history, especially mutable history shared by multiple developers.

        Setting up

        For these examples, we’re going to use a slightly different workflow: as before, Alice and Bob share a public repository. But this time there is no review repository. Instead, Alice and Bob put on their cowboy hats, throw good practice to the wind, and pull directly from each other’s working repositories.

        So we throw away everything except public and reclone: 

        $ rm -rf review alice bob
$ hg clone public alice
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone public bob
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved

        Once again we have to configure their repositories: enable evolve and (since Alice and Bob will be pulling directly from each other) make their repositories non-publishing. Edit Alice’s configuration: 

        $ hg -R alice config --edit --local

        and add 

        [extensions]
rebase =
evolve =

[phases]
publish = false

        Then edit Bob’s repository configuration: 

        $ hg -R bob config --edit --local

        and add the same text.

        Example 6: Divergent changesets

        When an obsolete changeset has two successors, those successors are divergent. One way to get into such a situation is by failing to communicate with your teammates. Let’s see how that might happen.

        First, we’ll have Bob commit a bug fix that could still be improved: 

        $ cd bob
$ echo 'pretty good fix' >> file1
$ hg commit -u bob -m 'fix bug 24 (v1)'                   # rev 4:2fe6

        Since Alice and Bob are now in cowboy mode, Alice pulls Bob’s draft changeset and amends it herself. 

        $ cd ../alice
$ hg pull -u ../bob
[...]
added 1 changesets with 1 changes to 1 files
$ echo 'better fix (alice)' >> file1
$ hg amend -u alice -m 'fix bug 24 (v2 by alice)'

        But Bob has no idea that Alice just did this. (See how important good communication is?) So he implements a better fix of his own: 

        $ cd ../bob
$ echo 'better fix (bob)' >> file1
$ hg amend -u bob -m 'fix bug 24 (v2 by bob)'             # rev 6:a360

        At this point, the divergence exists, but only in theory: Bob’s original changeset, 4:2fe6, is obsolete and has two successors. But those successors are in different repositories, so the trouble is not visible to anyone yet. It will be as soon as Bob pulls from Alice’s repository (or vice-versa). 

        $ hg pull ../alice
[...]
added 1 changesets with 1 changes to 2 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
2 new divergent changesets

        Figure 9 shows the situation in Bob’s repository.

        [figure SG09: Bob’s repo with 2 heads for the 2 divergent changesets, 6:a360 and 7:e3f9; wc is at 6:a360; both are successors of obsolete 4:2fe6, hence divergence]

        Now we need to get out of trouble. As usual, the answer is to evolve history. 

        $ HGMERGE=internal:other hg evolve
merge:[6] fix bug 24 (v2 by bob)
with: [7] fix bug 24 (v2 by alice)
base: [4] fix bug 24 (v1)
0 files updated, 1 files merged, 0 files removed, 0 files unresolved

        Figure 10 shows how Bob’s repository looks now.

        [figure SG10: only one visible head, 9:5ad6, successor to hidden 6:a360 and 7:e3f9]

        We carefully dodged a merge conflict by specifying a merge tool (internal:other) that will take Alice’s changes over Bob’s. (You might wonder why Bob wouldn’t prefer his own changes by using internal:local. He’s avoiding a bug in evolve that occurs when evolving divergent changesets using internal:local.)

        # XXX this link does not work .. bug: https://bitbucket.org/marmoute/mutable-history/issue/48/

        ** STOP HERE: WORK IN PROGRESS **

        Phase-divergence: when a rewritten changeset is made public

        If Alice and Bob are collaborating on some mutable changesets, it’s possible to get into a situation where an otherwise worthwhile changeset cannot be pushed to the public repository; it is phase-divergent with another changeset that was made public first. Let’s demonstrate one way this could happen.

        It starts with Alice committing a bug fix. Right now, we don’t yet know if this bug fix is good enough to push to the public repository, but it’s good enough for Alice to commit. 

        $ cd alice
$ echo 'fix' > file2
$ hg commit -A -m 'fix bug 15'
adding file2

        Now Bob has a bad idea: he decides to pull whatever Alice is working on and tweak her bug fix to his taste: 

        $ cd ../bob
$ hg pull -u ../alice
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ echo 'Fix.' > file2
$ hg amend -A -m 'fix bug 15 (amended)'

        (Note the lack of communication between Alice and Bob. Failing to communicate with your colleagues is a good way to get into trouble. Nevertheless, evolve can usually sort things out, as we will see.)

        [figure SG06: Bob’s repo with one amendment]

        After some testing, Alice realizes her bug fix is just fine as it is: no need for further polishing and amending, this changeset is ready to publish. 

        $ cd ../alice
$ hg push
[...]
added 1 changesets with 1 changes to 1 files

        This introduces a contradiction: in Bob’s repository, changeset 2:e011 (his copy of Alice’s fix) is obsolete, since Bob amended it. But in Alice’s repository (and the public repository), that changeset is public: it is immutable, carved in stone for all eternity. No changeset can be both obsolete and public, so Bob is in for a surprise the next time he pulls from public: 

        $ cd ../bob
$ hg pull -q -u
1 new phase-divergent changesets

        Figure 7 shows what just happened to Bob’s repository: changeset 2:e011 is now public, so it can’t be obsolete. When that changeset was obsolete, it made perfect sense for it to have a successor, namely Bob’s amendment of Alice’s fix (changeset 4:fe88). But it’s illogical for a public changeset to have a successor, so 4:fe88 is troubled: it has become bumped.

        [figure SG07: 2:e011 now public not obsolete, 4:fe88 now bumped]

        As usual when there’s trouble in your repository, the solution is to evolve it: 

        $ hg evolve --all

        Figure 8 illustrates Bob’s repository after evolving away the bumped changeset. Ignoring the obsolete changesets, Bob now has a nice, clean, simple history. His amendment of Alice’s bug fix lives on, as changeset 5:227d—albeit with a software-generated commit message. (Bob should probably amend that changeset to improve the commit message.) But the important thing is that his repository no longer has any troubled changesets, thanks to evolve.

        [figure SG08: 5:227d is new, formerly bumped changeset 4:fe88 now hidden]

        Conclusion

        Mutable history is a powerful tool. Like a sharp knife, an experienced user can do wonderful things with it, much more wonderful than with a dull knife (never mind a rusty spoon). At the same time, an inattentive or careless user can do harm to himself or others. Mercurial with evolve goes to great lengths to limit the harm you can do by trying to handle all possible types of “troubled” changesets. Nevertheless, having a first-aid kit nearby does not mean you should stop being careful with sharp knives.

        Mutable history shared across multiple repositories by a single developer is a natural extension of this model. Once you are used to using a single sharp knife on its own, it’s pretty straightforward to chop onions and mushrooms using the same knife, or to alternate between two chopping boards with different knives.

        Mutable history shared by multiple developers is a scary place to go. Imagine a professional kitchen full of expert chefs tossing their favourite knives back and forth, with the occasional axe or chainsaw thrown in to spice things up. If you’re confident that you and your colleagues can do it without losing a limb, go for it. But be sure to practice a lot first before you rely on it!

        «  Evolve: User Guide   ::   Contents   ::   Evolve: Concepts  »

        © Copyright 2010-2014, Pierre-Yves David, Greg Ward, and contributors. Last updated on Sep 21, 2017. Created using Sphinx 1.2.3.
        ================================================ FILE: test/test-pages/metadata-content-missing/expected-metadata.json ================================================ { "title": "My title", "byline": "Creator Name", "dir": null, "excerpt": "Preferred description", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/metadata-content-missing/expected.html ================================================

        Test document title

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        ================================================ FILE: test/test-pages/metadata-content-missing/source.html ================================================ Title Element

        Test document title

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        ================================================ FILE: test/test-pages/missing-paragraphs/expected-metadata.json ================================================ { "title": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt", "byline": "Henri Sivonen", "dir": null, "lang": "en", "excerpt": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy\n eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam\n voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit\n amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam\n nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,\n sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\n Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor\n sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed\n diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,\n sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\n Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor\n sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/missing-paragraphs/expected.html ================================================

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        ================================================ FILE: test/test-pages/missing-paragraphs/source.html ================================================ Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        Secondary header

        Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

        ================================================ FILE: test/test-pages/mozilla-1/expected-metadata.json ================================================ { "title": "Firefox — Customize and make it your own — The most flexible browser on the Web", "byline": null, "dir": "ltr", "lang": "en", "excerpt": "It’s easier than ever to personalize Firefox and make it work the way\n you do.\n No other browser gives you so much choice and flexibility.", "siteName": "Mozilla", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/mozilla-1/expected.html ================================================

        It’s easier than ever to personalize Firefox and make it work the way you do.
        No other browser gives you so much choice and flexibility.

        Designed to
        be redesigned

        Get fast and easy access to the features you use most in the new menu. Open the “Customize” panel to add, move or remove any button you want. Keep your favorite features — add-ons, private browsing, Sync and more — one quick click away.

        More ways to customize

        Themes

        Make Firefox match your style. Choose from thousands of themes and dress up your browser with a single click.

        Try it now
        Learn more

        Next

        Preview of the currently selected theme

        Add-ons

        Next

        Add-ons are like apps that you install to add features to Firefox. They let you compare prices, check the weather, listen to music, send a tweet and more.

        • Read the latest news & blogs
        • Manage your downloads
        • Watch videos & view photos

        Here are a few of our favorites
        Learn more

        Awesome Bar

        Next

        The Awesome Bar learns as you browse to make your version of Firefox unique. Find and return to your favorite sites without having to remember a URL.

        See what it can do for you

        Firefox Awesome Bar

        ================================================ FILE: test/test-pages/mozilla-1/source.html ================================================ Firefox — Customize and make it your own — The most flexible browser on the Web — Mozilla

        Make your Firefox your own

        It’s easier than ever to personalize Firefox and make it work the way you do.
        No other browser gives you so much choice and flexibility.

        Designed to
        be redesigned

        Get fast and easy access to the features you use most in the new menu. Open the “Customize” panel to add, move or remove any button you want. Keep your favorite features — add-ons, private browsing, Sync and more — one quick click away.

        More ways to customize

        Themes

        Make Firefox match your style. Choose from thousands of themes and dress up your browser with a single click.

        Try it now
        Learn more
        Preview of the currently selected theme

        Add-ons

        Add-ons are like apps that you install to add features to Firefox. They let you compare prices, check the weather, listen to music, send a tweet and more.

        • Read the latest news & blogs
        • Manage your downloads
        • Watch videos & view photos
        Here are a few of our favorites
        Learn more

        Awesome Bar

        The Awesome Bar learns as you browse to make your version of Firefox unique. Find and return to your favorite sites without having to remember a URL.

        See what it can do for you
        Firefox Awesome Bar

        Keep your Firefox in Sync

        Access your bookmarks, history, passwords and more from any device to
        make Firefox your own wherever you use it.

        Learn more about Sync
        Get help with Sync
        ================================================ FILE: test/test-pages/mozilla-2/expected-metadata.json ================================================ { "title": "Welcome to Firefox Developer Edition", "byline": null, "dir": "ltr", "lang": "en", "excerpt": "Built for those who build the Web. Introducing the only browser made for developers.", "siteName": "Mozilla", "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/mozilla-2/expected.html ================================================

        Get to know the features that make it the most complete browser for building the Web.

        • Screenshot

          WebIDE

          Develop, deploy and debug Firefox OS apps directly in your browser, or on a Firefox OS device, with this tool that replaces App Manager.

          Learn more about WebIDE
        • Screenshot

          Valence

          Develop and debug your apps across multiple browsers and devices with this powerful extension that comes pre-installed with Firefox Developer Edition.

          Learn more about Valence

        Important: Sync your new profile

        Developer Edition comes with a new profile so you can run it alongside other versions of Firefox. To access your bookmarks, browsing history and more, you need to sync the profile with your existing Firefox Account, or create a new one. Learn more

        Features and tools

        ================================================ FILE: test/test-pages/mozilla-2/source.html ================================================ Welcome to Firefox Developer Edition
        Mozilla

        Firefox Developer Edition

        Welcome to Firefox Developer Edition

        Get to know the features that make it the most complete browser for building the Web.

        • Screenshot

          WebIDE

          Develop, deploy and debug Firefox OS apps directly in your browser, or on a Firefox OS device, with this tool that replaces App Manager.

          Learn more about WebIDE
        • Screenshot

          Valence

          Develop and debug your apps across multiple browsers and devices with this powerful extension that comes pre-installed with Firefox Developer Edition.

          Learn more about Valence

        Important: Sync your new profile

        Developer Edition comes with a new profile so you can run it alongside other versions of Firefox. To access your bookmarks, browsing history and more, you need to sync the profile with your existing Firefox Account, or create a new one. Learn more

        Features and tools

        ================================================ FILE: test/test-pages/msn/expected-metadata.json ================================================ { "title": "Nintendo's first iPhone game will launch in December for $10", "byline": "Alex Perry", "dir": "ltr", "lang": "en-US", "excerpt": "Nintendo and Apple shocked the world earlier this year by announcing \"Super Mario Run,\" the legendary gaming company's first foray into mobile gaming. ", "siteName": "MSN", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/msn/expected.html ================================================

        <span style="font-size:13px;">Nintendo/Apple</span> © Provided by Business Insider Inc Nintendo/Apple Nintendo and Apple shocked the world earlier this year by announcing "Super Mario Run," the legendary gaming company's first foray into mobile gaming. It's a Mario game you can play on your phone with just one hand, so what's not to love?

        Thankfully, now we know when you can get it and for how much: "Super Mario Run" will launch on iPhone and iPad on December 15, for a flat fee of $9.99. You can play a sample of the game modes for free, but unlike most other mobile games that let you download for free but require money to keep playing, or access parts of the game, you can pay $10 to get everything right away.

        In case you haven't heard, "Super Mario Run" is essentially a regular, side-scrolling "Super Mario" game with one key difference: You don't control Mario's movement. He runs automatically and all you do is tap the screen to jump.

        The name and basic idea might sound like one of those endless score attack games like "Temple Run," but that's not the case. "Super Mario Run" is divided into hand-crafted levels with a clear end-point like any other Mario game, meaning you're essentially getting the Mario experience for $10 without needing to control his movement.

        $10 might seem like a bit much compared to the $0 people pay for most mobile games, but it's possible the game has $10 worth of levels to play in it. It's also not iPhone exclusive, but the Android version will launch at a later, currently unknown date.

        To see "Super Mario Run" in action, check out the footage below:

        ================================================ FILE: test/test-pages/msn/source.html ================================================ Nintendo's first iPhone game will launch in December for $10

        Nintendo's first iPhone game will launch in December for $10

        <span style="font-size:13px;">Nintendo/Apple</span> © Provided by Business Insider Inc Nintendo/Apple Nintendo and Apple shocked the world earlier this year by announcing "Super Mario Run," the legendary gaming company's first foray into mobile gaming. It's a Mario game you can play on your phone with just one hand, so what's not to love?

        Thankfully, now we know when you can get it and for how much: "Super Mario Run" will launch on iPhone and iPad on December 15, for a flat fee of $9.99. You can play a sample of the game modes for free, but unlike most other mobile games that let you download for free but require money to keep playing, or access parts of the game, you can pay $10 to get everything right away.

        In case you haven't heard, "Super Mario Run" is essentially a regular, side-scrolling "Super Mario" game with one key difference: You don't control Mario's movement. He runs automatically and all you do is tap the screen to jump.

        The name and basic idea might sound like one of those endless score attack games like "Temple Run," but that's not the case. "Super Mario Run" is divided into hand-crafted levels with a clear end-point like any other Mario game, meaning you're essentially getting the Mario experience for $10 without needing to control his movement.

        $10 might seem like a bit much compared to the $0 people pay for most mobile games, but it's possible the game has $10 worth of levels to play in it. It's also not iPhone exclusive, but the Android version will launch at a later, currently unknown date.

        To see "Super Mario Run" in action, check out the footage below:


        Intel
        What Were the Big Topics at Intel® Developer Forum 2016?Intel
        Undo
        My Snoring Solution
        Say Goodbye To Snoring, Say Hello to SleepMy Snoring Solution
        Undo
        Xtra-PC
        Slow Computer? You Don't Need to Buy a New One Thanks to This DiscoveryXtra-PC
        Undo
        Apple Inc. Has an iPhone 7 Plus Problem
        The Motley Fool on MSN
        Undo
        How to buy Nintendo's NES Classic
        The Verge on MSN
        Undo
        Apple's iPhone sales likely to continue falling, top analyst says
        CNBC on MSN
        Undo
        Babbel
        1,000,000 Have Adopted This New Way To Learn A Foreign LanguageBabbel
        Undo
        Stormfall: Free Online Game
        This Is The Most Addicting Game Of The Year!Stormfall: Free Online Game
        Undo
        Pirates: Free Online Game
        Find out why more than 20 million people can't stop playing this gamePirates: Free Online Game
        Undo
        Designers test out 'invincible' iPhone case by throwing phone around store
        Newsflare on MSN
        Undo
        Nintendo’s $59.99 NES Classic is in short supply, but you can buy one online for $1,000
        CNBC on MSN
        Undo
        Chinese consumers' body takes up complaints of iPhone shutdowns with Apple
        Reuters on MSN
        Undo
        Reuters TV
        U.S. gymnast tormented by bulliesReuters TV
        Undo
        Save70.com
        The Ultimate Way to Get Cheap Hotel RoomsSave70.com
        Undo
        HealthGrove — By Graphiq
        Here's Why These Are the Most Dangerous Countries for TravelHealthGrove — By Graphiq
        Undo
        Will Trump Be Impeached?
        Vanity Fair on MSN
        Undo
        Cause of death released for missing teen found slain
        CBS News on MSN
        Undo
        Nintendo's mini NES is out today
        Engadget on MSN
        Undo
        Microsoft Store
        Want to play Xbox games on almost any device? Learn howMicrosoft Store
        Undo
        Microsoft Store
        Now Shipping! Buy Madden Xbox One S now and get loads of extrasMicrosoft Store
        Undo
        Microsoft Store
        Stream your night away with the best devices to watch your favorite showsMicrosoft Store
        Undo
        Go to MSN Home
        Go to MSN News
        Feedback

        Report inappropriate content

        Sorry!
        There was a problem. Please try again.

        Help us maintain a healthy and vibrant community by reporting any illegal or inappropriate behavior that violates MSN’s Code of Conduct.

        Leave your comment

        Found the story interesting?

        Like us on Facebook to see similar stories


        Send Feedback

        We appreciate your input!

        • I'm having problems with Top Destinations

        • I'm having issues searching

        • I'm having problems with Featured Apps

        • My Topics feedback

        • Other

        Please give an overall site rating:

        Make MSN my homepage

        • On the toolbar, click Firefox Then, click Preferences.
          On the toolbar, click Firefox Then, click Preferences.
        • In the Preferences window, select General.
          In the Preferences window, select General.
        • In the text box next to Home page, simply type www.msn.com.
          In the text box next to Home page, simply type www.msn.com.
        ================================================ FILE: test/test-pages/normalize-spaces/expected-metadata.json ================================================ { "title": "Normalize space test", "byline": null, "dir": null, "excerpt": "Lorem\n ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n\ttab here\n incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/normalize-spaces/expected.html ================================================

        Lorem

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tab here incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Foo

        Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        ================================================ FILE: test/test-pages/normalize-spaces/source.html ================================================ Normalize space test

        Lorem

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tab here incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Foo

        Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        ================================================ FILE: test/test-pages/nytimes-1/expected-metadata.json ================================================ { "title": "United States to Lift Sudan Sanctions", "byline": "Jeffrey Gettleman", "dir": null, "lang": "en", "excerpt": "For the first time since the 1990s, the country will be able to trade extensively with the United States.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/nytimes-1/expected.html ================================================
        Photo

        United Nations peacekeepers at a refugee camp in Sudan on Monday. In exchange for the lifting of United States trade sanctions, Sudan has said it will improve access for aid groups, stop supporting rebels in neighboring South Sudan and cooperate with American intelligence agents. Credit Ashraf Shazly/Agence France-Presse — Getty Images

        LONDON — After nearly 20 years of hostile relations, the American government plans to reverse its position on Sudan and lift trade sanctions, Obama administration officials said late Thursday.

        Sudan is one of the poorest, most isolated and most violent countries in Africa, and for years the United States has imposed punitive measures against it in a largely unsuccessful attempt to get the Sudanese government to stop killing its own people.

        On Friday, the Obama administration will announce a new Sudan strategy. For the first time since the 1990s, the nation will be able to trade extensively with the United States, allowing it to buy goods like tractors and spare parts and attract much-needed investment in its collapsing economy.

        In return, Sudan will improve access for aid groups, stop supporting rebels in neighboring South Sudan, cease the bombing of insurgent territory and cooperate with American intelligence agents.

        American officials said Sudan had already shown important progress on a number of these fronts. But to make sure the progress continues, the executive order that President Obama plans to sign on Friday, days before leaving office, will have a six-month review period. If Sudan fails to live up to its commitments, the embargo can be reinstated.

        Analysts said good relations with Sudan could strengthen moderate voices within the country and give the Sudanese government incentives to refrain from the brutal tactics that have defined it for decades.

        In 1997, President Bill Clinton imposed a comprehensive trade embargo against Sudan and blocked the assets of the Sudanese government, which was suspected of sponsoring international terrorism. In the mid-1990s, Osama bin Laden lived in Khartoum, the capital, as a guest of Sudan’s government.

        In 1998, Bin Laden’s agents blew up the United States Embassies in Kenya and Tanzania, killing more than 200 people. In retaliation, Mr. Clinton ordered a cruise missile strike against a pharmaceutical factory in Khartoum.

        Since then, American-Sudanese relations have steadily soured. The conflict in Darfur, a vast desert region of western Sudan, was a low point. After rebels in Darfur staged an uprising in 2003, Sudanese security services and their militia allies slaughtered tens of thousands of civilians, leading to condemnation around the world, genocide charges at the International Criminal Court against Sudan’s president, Omar Hassan al-Bashir, and a new round of American sanctions.

        American officials said Thursday that the American demand that Mr. Bashir be held accountable had not changed. Neither has Sudan’s status as one of the few countries, along with Iran and Syria, that remain on the American government’s list of state sponsors of terrorism.

        Sales of military equipment will still be prohibited, and some Sudanese militia and rebel leaders will still face sanctions.

        But the Obama administration is clearly trying to open a door to Sudan. There is intense discontent across the country, and its economy is imploding. American officials have argued for years that it was time to help Sudan dig itself out of the hole it had created.

        Officials divulged Thursday that the Sudanese government had allowed two visits by American operatives to a restricted border area near Libya, which they cited as evidence of a new spirit of cooperation on counterterrorism efforts.

        In addition to continuing violence in Darfur, several other serious conflicts are raging in southern and central Sudan, along with a civil war in newly independent South Sudan, which Sudan has been suspected of inflaming with covert arms shipments.

        Eric Reeves, one of the leading American academic voices on Sudan, said he was “appalled” that the American government was lifting sanctions.

        He said that Sudan’s military-dominated government continued to commit grave human rights abuses and atrocities, and he noted that just last week Sudanese security services killed more than 10 civilians in Darfur.

        “There is no reason to believe the guys in charge have changed their stripes,” said Mr. Reeves, a senior fellow at the François-Xavier Bagnoud Center for Health and Human Rights at Harvard University. “These guys are the worst of the worst.”

        Obama administration officials said that they had briefed President-elect Donald J. Trump’s transition team, but that they did not know if Mr. Trump would stick with a policy of warmer relations with Sudan.

        They said that Sudan had a long way to go in terms of respecting human rights, but that better relations could help increase American leverage.

        Mr. Reeves said he thought that the American government was being manipulated and that the Obama administration had made a “deal with the devil.”

        Continue reading the main story

        ================================================ FILE: test/test-pages/nytimes-1/source.html ================================================ United States to Lift Sudan Sanctions - The New York Times

        Africa

        United States to Lift Sudan Sanctions

        By JEFFREY GETTLEMAN

        United Nations peacekeepers at a refugee camp in Sudan on Monday. In exchange for the lifting of United States trade sanctions, Sudan has said it will improve access for aid groups, stop supporting rebels in neighboring South Sudan and cooperate with American intelligence agents. Ashraf Shazly/Agence France-Presse — Getty Images

        LONDON — After nearly 20 years of hostile relations, the American government plans to reverse its position on Sudan and lift trade sanctions, Obama administration officials said late Thursday.

        Sudan is one of the poorest, most isolated and most violent countries in Africa, and for years the United States has imposed punitive measures against it in a largely unsuccessful attempt to get the Sudanese government to stop killing its own people.

        On Friday, the Obama administration will announce a new Sudan strategy. For the first time since the 1990s, the nation will be able to trade extensively with the United States, allowing it to buy goods like tractors and spare parts and attract much-needed investment in its collapsing economy.

        In return, Sudan will improve access for aid groups, stop supporting rebels in neighboring South Sudan, cease the bombing of insurgent territory and cooperate with American intelligence agents.

        American officials said Sudan had already shown important progress on a number of these fronts. But to make sure the progress continues, the executive order that President Obama plans to sign on Friday, days before leaving office, will have a six-month review period. If Sudan fails to live up to its commitments, the embargo can be reinstated.

        Analysts said good relations with Sudan could strengthen moderate voices within the country and give the Sudanese government incentives to refrain from the brutal tactics that have defined it for decades.

        In 1997, President Bill Clinton imposed a comprehensive trade embargo against Sudan and blocked the assets of the Sudanese government, which was suspected of sponsoring international terrorism. In the mid-1990s, Osama bin Laden lived in Khartoum, the capital, as a guest of Sudan’s government.

        In 1998, Bin Laden’s agents blew up the United States Embassies in Kenya and Tanzania, killing more than 200 people. In retaliation, Mr. Clinton ordered a cruise missile strike against a pharmaceutical factory in Khartoum.

        Since then, American-Sudanese relations have steadily soured. The conflict in Darfur, a vast desert region of western Sudan, was a low point. After rebels in Darfur staged an uprising in 2003, Sudanese security services and their militia allies slaughtered tens of thousands of civilians, leading to condemnation around the world, genocide charges at the International Criminal Court against Sudan’s president, Omar Hassan al-Bashir, and a new round of American sanctions.

        American officials said Thursday that the American demand that Mr. Bashir be held accountable had not changed. Neither has Sudan’s status as one of the few countries, along with Iran and Syria, that remain on the American government’s list of state sponsors of terrorism.

        Sales of military equipment will still be prohibited, and some Sudanese militia and rebel leaders will still face sanctions.

        But the Obama administration is clearly trying to open a door to Sudan. There is intense discontent across the country, and its economy is imploding. American officials have argued for years that it was time to help Sudan dig itself out of the hole it had created.

        Officials divulged Thursday that the Sudanese government had allowed two visits by American operatives to a restricted border area near Libya, which they cited as evidence of a new spirit of cooperation on counterterrorism efforts.

        In addition to continuing violence in Darfur, several other serious conflicts are raging in southern and central Sudan, along with a civil war in newly independent South Sudan, which Sudan has been suspected of inflaming with covert arms shipments.

        Eric Reeves, one of the leading American academic voices on Sudan, said he was “appalled” that the American government was lifting sanctions.

        He said that Sudan’s military-dominated government continued to commit grave human rights abuses and atrocities, and he noted that just last week Sudanese security services killed more than 10 civilians in Darfur.

        “There is no reason to believe the guys in charge have changed their stripes,” said Mr. Reeves, a senior fellow at the François-Xavier Bagnoud Center for Health and Human Rights at Harvard University. “These guys are the worst of the worst.”

        Obama administration officials said that they had briefed President-elect Donald J. Trump’s transition team, but that they did not know if Mr. Trump would stick with a policy of warmer relations with Sudan.

        They said that Sudan had a long way to go in terms of respecting human rights, but that better relations could help increase American leverage.

        Mr. Reeves said he thought that the American government was being manipulated and that the Obama administration had made a “deal with the devil.”

        A version of this article appears in print on January 13, 2017, on Page A7 of the New York edition with the headline: Sanctions on Sudan Will Be Lifted. Order Reprints| Today's Paper|Subscribe

        Go to Home Page »

        The New York Times

        ================================================ FILE: test/test-pages/nytimes-2/expected-metadata.json ================================================ { "title": "Yahoo’s Sale to Verizon Leaves Shareholders With Little Say", "byline": "Steven Davidoff Solomon", "dir": null, "lang": "en", "excerpt": "The internet giant’s decision to sell its business is plagued with challenges that reveal how unusual deal structures can affect shareholders.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/nytimes-2/expected.html ================================================
        Photo

        Credit Harry Campbell

        Yahoo’s $4.8 billion sale to Verizon is a complicated beast, showing how different acquisition structures can affect how shareholders are treated.

        First, let’s say what the Yahoo sale is not. It is not a sale of the publicly traded company. Instead, it is a sale of the Yahoo subsidiary and some related assets to Verizon.

        The sale is being done in two steps. The first step will be the transfer of any assets related to Yahoo business to a singular subsidiary. This includes the stock in the business subsidiaries that make up Yahoo that are not already in the single subsidiary, as well as the odd assets like benefit plan rights. This is what is being sold to Verizon. A license of Yahoo’s oldest patents is being held back in the so-called Excalibur portfolio. This will stay with Yahoo, as will Yahoo’s stakes in Alibaba Group and Yahoo Japan.

        It is hard to overestimate how complex an asset sale like this is. Some of the assets are self-contained, but they must be gathered up and transferred. Employees need to be shuffled around and compensation arrangements redone. Many contracts, like the now-infamous one struck with the search engine Mozilla, which may result in a payment of up to a $1 billion, will contain change-of-control provisions that will be set off and have to be addressed. Tax issues always loom large.

        Continue reading the main story

        Continue reading the main story

        In the second step, at the closing, Yahoo will sell the stock in the single subsidiary to Verizon. At that point, Yahoo will change its name to something without “Yahoo” in it. My favorite is simply Remain Co., the name Yahoo executives are using. Remain Co. will become a holding company for the Alibaba and Yahoo Japan stock. Included will also be $10 billion in cash, plus the Excalibur patent portfolio and a number of minority investments including Snapchat. Ahh, if only Yahoo had bought Snapchat instead of Tumblr (indeed, if only Yahoo had bought Google or Facebook when it had the chance).

        Because it is a sale of a subsidiary, the $4.8 billion will be paid to Yahoo. Its shareholders will not receive any money unless Yahoo pays it out in a dividend (after paying taxes). Instead, Yahoo shareholders will be left holding shares in the renamed company.

        Verizon’s Yahoo will then be run under the same umbrella as AOL. It is unclear whether there will be a further merger of the two businesses after the acquisition. Plans for Yahoo are still a bit in flux in part because of the abnormal sale process.

        As for Remain Co., it will become a so-called investment company. This is a special designation for a company that holds securities for investment but does not operate a working business. Investment companies are subject to special regulation under the Investment Company Act of 1940. Remain Co. will probably just sit there, returning cash to shareholders and waiting for Alibaba to buy it in a tax-free transaction. (Alibaba says it has no plans to do this, but most people do not believe this).

        The rights of Yahoo shareholders in this sale will be different from those in an ordinary sale, when an entire company is bought.

        Ordinary sales are done in one of two ways: in a merger where the target is merged into a subsidiary of the buyer and the target shareholders receive the cash (or other consideration), or in a tender offer that gives the target shareholders a choice to tender into the offer or not. Then there will be a merger where the target is merged into the buyer’s subsidiary and the target shareholders are forcibly squeezed out, receiving the merger consideration. (if you want to know why you would choose one structure over another, I wrote a good primer in 2009.)

        In either case, shareholders get a say. They either vote on the merger or decide whether to tender into the offer.

        In both cases, there would be appraisal rights if the buyer pays cash. This means that shareholders can object to the deal by not voting for it or not tendering into the offer and instead asking a court to value their shares – this is what happened in Dell’s buyout in 2013.

        The Yahoo deal, however, is not a sale of the public company. It is an asset sale, in which there is only a shareholder vote if there is a sale of “all” or “substantially all” of the assets of the company. Yahoo is not all of the assets or even “substantially all” – the Alibaba shares being left behind in Remain Co. are worth about $28 billion, or six times the value of the cash Verizon is paying for the Yahoo assets it is buying.

        The courts have held that the definition of “substantially all” includes a change of business in a company because of an asset sale where the assets are “qualitatively vital.” And that certainly applies here. So there will be a vote – indeed, Yahoo has no problem with a vote – and shareholders are desperate to sell at this point.

        There will be no appraisal rights, however. Again, in an asset sale, there are no appraisal rights. So anyone who votes against the deal and thinks this is a bum price is out of luck.

        The different standards for voting and appraisal rights apply because the structure of the deal is a quirk of the law in Delaware, where Yahoo is incorporated, that allows lawyers to sometimes work around these issues simply by changing the way a deal is done.

        In Yahoo’s case, this is not deliberate, though. It is simply the most expedient way to get rid of the assets.

        Whether this is the most tax-efficient way is unclear to me as a nontax lawyer (email me if you know). Yahoo is likely to have a tax bill on the sale, possibly a substantial one. And I presume there were legal reasons for not using a Morris Trust structure, in which Yahoo would have been spun off and immediately sold to Verizon so that only Yahoo’s shareholders paid tax on the deal. In truth, the Yahoo assets being sold are only about 10 percent of the value of the company, so the time and logistics for such a sale when Yahoo is a melting ice cube may not have been worth it.

        Finally, if another bidder still wants to acquire Yahoo, it has time. The agreement with Verizon allows Yahoo to terminate the deal and accept a superior offer by paying a $144 million breakup fee to Verizon. And if Yahoo shareholders change their minds and want to stick with Yahoo’s chief executive, Marissa Mayer, and vote down the deal, there is a so-called naked no-vote termination fee of $15 million payable to Verizon to reimburse expenses.

        All in all, this was as hairy a deal as they come. There was the procedural and logistical complications of selling a company when the chief executive wanted to stay. Then there was the fact that this was an asset sale, including all of the challenges that go with it. Throw in all of the tax issues and the fact that this is a public company, and it is likely that the lawyers involved will have nightmares for years to come.

        Continue reading the main story

        ================================================ FILE: test/test-pages/nytimes-2/source.html ================================================ Yahoo’s Sale to Verizon Leaves Shareholders With Little Say - The New York Times

        Yahoo’s Sale to Verizon Leaves Shareholders With Little Say

        Deal Professor

        By STEVEN DAVIDOFF SOLOMON

        Harry Campbell

        Yahoo’s $4.8 billion sale to Verizon is a complicated beast, showing how different acquisition structures can affect how shareholders are treated.

        First, let’s say what the Yahoo sale is not. It is not a sale of the publicly traded company. Instead, it is a sale of the Yahoo subsidiary and some related assets to Verizon.

        The sale is being done in two steps. The first step will be the transfer of any assets related to Yahoo business to a singular subsidiary. This includes the stock in the business subsidiaries that make up Yahoo that are not already in the single subsidiary, as well as the odd assets like benefit plan rights. This is what is being sold to Verizon. A license of Yahoo’s oldest patents is being held back in the so-called Excalibur portfolio. This will stay with Yahoo, as will Yahoo’s stakes in Alibaba Group and Yahoo Japan.

        It is hard to overestimate how complex an asset sale like this is. Some of the assets are self-contained, but they must be gathered up and transferred. Employees need to be shuffled around and compensation arrangements redone. Many contracts, like the now-infamous one struck with the search engine Mozilla, which may result in a payment of up to a $1 billion, will contain change-of-control provisions that will be set off and have to be addressed. Tax issues always loom large.

        In the second step, at the closing, Yahoo will sell the stock in the single subsidiary to Verizon. At that point, Yahoo will change its name to something without “Yahoo” in it. My favorite is simply Remain Co., the name Yahoo executives are using. Remain Co. will become a holding company for the Alibaba and Yahoo Japan stock. Included will also be $10 billion in cash, plus the Excalibur patent portfolio and a number of minority investments including Snapchat. Ahh, if only Yahoo had bought Snapchat instead of Tumblr (indeed, if only Yahoo had bought Google or Facebook when it had the chance).

        Because it is a sale of a subsidiary, the $4.8 billion will be paid to Yahoo. Its shareholders will not receive any money unless Yahoo pays it out in a dividend (after paying taxes). Instead, Yahoo shareholders will be left holding shares in the renamed company.

        Verizon’s Yahoo will then be run under the same umbrella as AOL. It is unclear whether there will be a further merger of the two businesses after the acquisition. Plans for Yahoo are still a bit in flux in part because of the abnormal sale process.

        As for Remain Co., it will become a so-called investment company. This is a special designation for a company that holds securities for investment but does not operate a working business. Investment companies are subject to special regulation under the Investment Company Act of 1940. Remain Co. will probably just sit there, returning cash to shareholders and waiting for Alibaba to buy it in a tax-free transaction. (Alibaba says it has no plans to do this, but most people do not believe this).

        The rights of Yahoo shareholders in this sale will be different from those in an ordinary sale, when an entire company is bought.

        Ordinary sales are done in one of two ways: in a merger where the target is merged into a subsidiary of the buyer and the target shareholders receive the cash (or other consideration), or in a tender offer that gives the target shareholders a choice to tender into the offer or not. Then there will be a merger where the target is merged into the buyer’s subsidiary and the target shareholders are forcibly squeezed out, receiving the merger consideration. (if you want to know why you would choose one structure over another, I wrote a good primer in 2009.)

        In either case, shareholders get a say. They either vote on the merger or decide whether to tender into the offer.

        In both cases, there would be appraisal rights if the buyer pays cash. This means that shareholders can object to the deal by not voting for it or not tendering into the offer and instead asking a court to value their shares – this is what happened in Dell’s buyout in 2013.

        The Yahoo deal, however, is not a sale of the public company. It is an asset sale, in which there is only a shareholder vote if there is a sale of “all” or “substantially all” of the assets of the company. Yahoo is not all of the assets or even “substantially all” – the Alibaba shares being left behind in Remain Co. are worth about $28 billion, or six times the value of the cash Verizon is paying for the Yahoo assets it is buying.

        The courts have held that the definition of “substantially all” includes a change of business in a company because of an asset sale where the assets are “qualitatively vital.” And that certainly applies here. So there will be a vote – indeed, Yahoo has no problem with a vote – and shareholders are desperate to sell at this point.

        There will be no appraisal rights, however. Again, in an asset sale, there are no appraisal rights. So anyone who votes against the deal and thinks this is a bum price is out of luck.

        The different standards for voting and appraisal rights apply because the structure of the deal is a quirk of the law in Delaware, where Yahoo is incorporated, that allows lawyers to sometimes work around these issues simply by changing the way a deal is done.

        In Yahoo’s case, this is not deliberate, though. It is simply the most expedient way to get rid of the assets.

        Whether this is the most tax-efficient way is unclear to me as a nontax lawyer (email me if you know). Yahoo is likely to have a tax bill on the sale, possibly a substantial one. And I presume there were legal reasons for not using a Morris Trust structure, in which Yahoo would have been spun off and immediately sold to Verizon so that only Yahoo’s shareholders paid tax on the deal. In truth, the Yahoo assets being sold are only about 10 percent of the value of the company, so the time and logistics for such a sale when Yahoo is a melting ice cube may not have been worth it.

        Finally, if another bidder still wants to acquire Yahoo, it has time. The agreement with Verizon allows Yahoo to terminate the deal and accept a superior offer by paying a $144 million breakup fee to Verizon. And if Yahoo shareholders change their minds and want to stick with Yahoo’s chief executive, Marissa Mayer, and vote down the deal, there is a so-called naked no-vote termination fee of $15 million payable to Verizon to reimburse expenses.

        All in all, this was as hairy a deal as they come. There was the procedural and logistical complications of selling a company when the chief executive wanted to stay. Then there was the fact that this was an asset sale, including all of the challenges that go with it. Throw in all of the tax issues and the fact that this is a public company, and it is likely that the lawyers involved will have nightmares for years to come.

        Go to Home Page »

        The New York Times

        ================================================ FILE: test/test-pages/nytimes-3/expected-metadata.json ================================================ { "title": "Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C.", "byline": "Corey Kilgannon", "dir": null, "lang": "en", "excerpt": "New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/nytimes-3/expected.html ================================================

        New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse.

        Image

        A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.CreditCreditChang W. Lee/The New York Times

        Corey Kilgannon

        [What you need to know to start the day: Get New York Today in your inbox.]

        A series of recent manhole fires in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City.

        Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers.

        The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks.

        And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse.

        In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city.

        “You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added.

        Image

        In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.CreditKirsten Luce for The New York Times

        When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air.

        For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids.

        To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said.

        “It also lets smoke escape, which can tip off the public to notify the authorities,” he added.

        Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter.

        Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said.

        Image

        There are typically between 400 and 600 water main breaks each year in New York City, an official said.CreditMichael Appleton for The New York Times

        The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter.

        Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call.

        During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s water system.

        “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said.

        Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle.

        Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours.

        The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said.

        Image

        In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.CreditFred R. Conrad/The New York Times

        Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University.

        But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year.

        The city is also replacing older, leak-prone water and sewer mains.

        Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron.

        Image

        Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.CreditHiroko Masuike/The New York Times

        Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction.

        Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said.

        And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added.

        In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said.

        Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades.

        Image

        A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.CreditÃngel Franco/The New York Times

        Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes.

        And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014.

        Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief.

        Simply put, more freeze-thaw cycles result in more potholes, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60.

        Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street.

        Image

        Workers learning how to fix water main breaks at a training center in Queens.CreditChang W. Lee/The New York Times

        Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills.

        Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water.

        “Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens.

        “I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’”

        Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the Character Study column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. @coreykilgannon • Facebook

        A version of this article appears in print on , on Page A22 of the New York edition with the headline: Under the City’s Streets, A Battle Against Winter. Order Reprints | Today’s Paper | Subscribe

        ================================================ FILE: test/test-pages/nytimes-3/source.html ================================================ Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C. - The New York Times
        Skip to contentSkip to site index
        New York
        Log In
        Today’s Paper

        Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C.

        New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse.

        Image
        A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.CreditCreditChang W. Lee/The New York Times
        Corey Kilgannon

        By Corey Kilgannon

        [What you need to know to start the day: Get New York Today in your inbox.]

        A series of recent manhole fires in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City.

        Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers.

        The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks.

        And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse.

        In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city.

        “You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added.

        Image
        In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.CreditKirsten Luce for The New York Times

        When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air.

        For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids.

        To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said.

        “It also lets smoke escape, which can tip off the public to notify the authorities,” he added.

        Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter.

        Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said.

        Image
        There are typically between 400 and 600 water main breaks each year in New York City, an official said.CreditMichael Appleton for The New York Times

        The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter.

        Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call.

        During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s water system.

        “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said.

        Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle.

        Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours.

        The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said.

        Image
        In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.CreditFred R. Conrad/The New York Times

        Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University.

        But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year.

        The city is also replacing older, leak-prone water and sewer mains.

        Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron.

        Image
        Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.CreditHiroko Masuike/The New York Times

        Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction.

        Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said.

        And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added.

        In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said.

        Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades.

        Image
        A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.CreditÃngel Franco/The New York Times

        Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes.

        And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014.

        Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief.

        Simply put, more freeze-thaw cycles result in more potholes, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60.

        Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street.

        Image
        Workers learning how to fix water main breaks at a training center in Queens.CreditChang W. Lee/The New York Times

        Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills.

        Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water.

        “Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens.

        “I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’”

        Why Are New York City’s Streets Always Under Construction?

        Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the Character Study column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. @coreykilgannon • Facebook

        A version of this article appears in print on , on Page A22 of the New York edition with the headline: Under the City’s Streets, A Battle Against Winter. Order Reprints | Today’s Paper | Subscribe

        Advertisement

        ================================================ FILE: test/test-pages/nytimes-4/expected-metadata.json ================================================ { "title": "As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military", "byline": "Nelson D. Schwartz", "dir": null, "lang": "en", "excerpt": "Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/nytimes-4/expected.html ================================================

        Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.

        Image

        Interest payments on the federal debt could surpass the Defense Department budget in 2023.CreditCreditJeon Heon-Kyun/EPA, via Shutterstock

        The federal government could soon pay more in interest on its debt than it spends on the military, Medicaid or children’s programs.

        The run-up in borrowing costs is a one-two punch brought on by the need to finance a fast-growing budget deficit, worsened by tax cuts and steadily rising interest rates that will make the debt more expensive.

        With less money coming in and more going toward interest, political leaders will find it harder to address pressing needs like fixing crumbling roads and bridges or to make emergency moves like pulling the economy out of future recessions.

        Within a decade, more than $900 billion in interest payments will be due annually, easily outpacing spending on myriad other programs. Already the fastest-growing major government expense, the cost of interest is on track to hit $390 billion next year, nearly 50 percent more than in 2017, according to the Congressional Budget Office.

        “It’s very much something to worry about,” said C. Eugene Steuerle, a fellow at the Urban Institute and a co-founder of the Urban-Brookings Tax Policy Center in Washington. “Everything else is getting squeezed.”

        Gradually rising interest rates would have made borrowing more expensive even without additional debt. But the tax cuts passed late last year have created a deeper hole, with the deficit increasing faster than expected. A budget bill approved in February that raised spending by $300 billion over two years will add to the financial pressure.

        The deficit is expected to total nearly $1 trillion next year — the first time it has been that big since 2012, when the economy was still struggling to recover from the financial crisis and interest rates were near zero.

        Deficit hawks have gone silent, even proposing changes that would exacerbate the deficit. House Republicans introduced legislation this month that would make the tax cuts permanent.

        “The issue has just disappeared,” said Senator Mark Warner, a Virginia Democrat. “There’s collective amnesia.”

        The combination, say economists, marks a journey into mostly uncharted financial territory.

        In the past, government borrowing expanded during recessions and waned in recoveries. That countercyclical policy has been a part of the standard Keynesian toolbox to combat downturns since the Great Depression.

        The deficit is soaring now as the economy booms, meaning the stimulus is pro-cyclical. The risk is that the government would have less room to maneuver if the economy slows.

        Aside from wartime or a deep downturn like the 1930s or 2008-9, “this sort of aggressive fiscal stimulus is unprecedented in U.S. history,” said Jeffrey Frankel, an economist at Harvard.

        Pouring gasoline on an already hot economy has resulted in faster growth — the economy expanded at an annualized rate of 4.2 percent in the second quarter. But Mr. Frankel warns that when the economy weakens, the government will find it more difficult to cut taxes or increase spending.

        Lawmakers might, in fact, feel compelled to cut spending as tax revenue falls, further depressing the economy. “There will eventually be another recession, and this increases the chances we will have to slam on the brakes when the car is already going too slowly,” Mr. Frankel said.

        Finding the money to pay investors who hold government debt will crimp other parts of the budget. In a decade, interest on the debt will eat up 13 percent of government spending, up from 6.6 percent in 2017.

        “By 2020, we will spend more on interest than we do on kids, including education, food stamps and aid to families,” said Marc Goldwein, senior policy director at the Committee for a Responsible Federal Budget, a research and advocacy organization.

        Interest costs already dwarf spending on many popular programs. For example, grants to students from low-income families for college total roughly $30 billion — about one-tenth of what the government will pay in interest this year. Interest payments will overtake Medicaid in 2020 and the Department of Defense budget in 2023.

        What’s more, the heavy burden of interest payments could make it harder for the government to repair aging infrastructure or take on other big new projects.

        Mr. Trump has called for spending $1 trillion on infrastructure, but Congress has not taken up that idea.

        More about the federal debt and the economy

        Until recently, ultralow interest rates, set by the Federal Reserve to support the economy, allowed lawmakers to borrow without fretting too much about the cost of that debt.

        But as the economy has strengthened, the Fed has gradually raised rates, starting in December 2015. The central bank is expected to push rates up again on Wednesday, and more increases are in store.

        “When rates went down to record lows, it allowed the government to take on more debt without paying more interest,” Mr. Goldwein said. “That party is ending.”

        Since the beginning of the year, the yield on the 10-year Treasury note has risen by more than half a percentage point, to 3.1 percent. The Congressional Budget Office estimates that the yield will climb to 4.2 percent in 2021. Given that the total public debt of the United States stands at nearly $16 trillion, even a small uptick in rates can cost the government billions.

        There’s no guarantee that these forecasts will prove accurate. If the economy weakens, rates might fall or rise only slightly, reducing interest payments. But rates could also overshoot the budget office forecast.

        Some members of Congress want to set the stage for even more red ink. Republicans in the House want to make last year’s tax cuts permanent, instead of letting some of them expire at the end of 2025. That would reduce federal revenue by an additional $631 billion over 10 years, according to the Tax Policy Center.

        Deficit hawks have warned for years that a day of reckoning is coming, exposing the United States to the kind of economic crisis that overtook profligate borrowers in the past like Greece or Argentina.

        But most experts say that isn’t likely because the dollar is the world’s reserve currency. As a result, the United States still has plenty of borrowing capacity left because the Fed can print money with fewer consequences than other central banks.

        And interest rates plunged over the last decade, even as the government turned to the market for trillions each year after the recession. That’s because Treasury bonds are still the favored port of international investors in any economic storm.

        “We exported a financial crisis a decade ago, and the world responded by sending us money,” said William G. Gale, a senior fellow at the Brookings Institution.

        But that privileged position has allowed politicians in both parties to avoid politically painful steps like cutting spending or raising taxes.

        That doesn’t mean rapidly rising interest costs and a bigger deficit won’t eventually catch up with us.

        Charles Schultze, chairman of the Council of Economic Advisers in the Carter administration, once summed up the danger of deficits with a metaphor. “It’s not so much a question of the wolf at the door, but termites in the woodwork.”

        Rather than simply splitting along party lines, lawmakers’ attitudes toward the deficit also depend on which party is in power. Republicans pilloried the Obama administration for proposing a large stimulus in the depths of the recession in 2009 and complained about the deficit for years.

        In 2013, Senator Mitch McConnell of Kentucky called the debt and deficit “the transcendent issue of our era.” By 2017, as Senate majority leader, he quickly shepherded the tax cut through Congress.

        Senator James Lankford, an Oklahoma Republican who warned of the deficit’s dangers in the past, nevertheless played down that threat on the Senate floor as the tax billed neared passage.

        “I understand it’s a risk, but I think it’s an appropriate risk to be able to say let’s allow Americans to keep more of their own money to invest in this economy,” he said.

        He also claimed the tax cuts would pay for themselves even as the Congressional Budget Office estimated that they would add $250 billion to the deficit on average from 2019 to 2024.

        In an interview, Mr. Lankford insisted that the jury was still out on whether the tax cuts would generate additional revenue, citing the strong economic growth recently.

        While the Republican about-face has been much more striking, Democrats have adjusted their position, too.

        Mr. Warner, the Virginia Democrat, called last year’s tax bill “the worst piece of legislation we have passed since I arrived in the Senate.” In 2009, however, when Congress passed an $800 billion stimulus bill backed by the Obama administration, he called it “a responsible mix of tax cuts and investments that will create jobs.”

        The difference, Mr. Warner said, was that the economy was near the precipice then.

        “There was virtual unanimity among economists that we needed a stimulus,” he said. “But a $2 trillion tax cut at the end of a business cycle with borrowed money won’t end well.”

        Nelson D. Schwartz has covered economics since 2012. Previously, he wrote about Wall Street and banking, and also served as European economic correspondent in Paris. He joined The Times in 2007 as a feature writer for the Sunday Business section. @NelsonSchwartz

        A version of this article appears in print on , on Page A1 of the New York edition with the headline: What May Soon Exceed Cost of U.S. Military? Interest on U.S. Debt . Order Reprints | Today’s Paper | Subscribe

        ================================================ FILE: test/test-pages/nytimes-4/source.html ================================================ As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military - The New York Times
        Skip to contentSkip to site index
        Economy
        Log In
        Today’s Paper

        As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military

        Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.

        Image
        Interest payments on the federal debt could surpass the Defense Department budget in 2023.CreditCreditJeon Heon-Kyun/EPA, via Shutterstock

        By Nelson D. Schwartz

        The federal government could soon pay more in interest on its debt than it spends on the military, Medicaid or children’s programs.

        The run-up in borrowing costs is a one-two punch brought on by the need to finance a fast-growing budget deficit, worsened by tax cuts and steadily rising interest rates that will make the debt more expensive.

        With less money coming in and more going toward interest, political leaders will find it harder to address pressing needs like fixing crumbling roads and bridges or to make emergency moves like pulling the economy out of future recessions.

        Within a decade, more than $900 billion in interest payments will be due annually, easily outpacing spending on myriad other programs. Already the fastest-growing major government expense, the cost of interest is on track to hit $390 billion next year, nearly 50 percent more than in 2017, according to the Congressional Budget Office.

        “It’s very much something to worry about,” said C. Eugene Steuerle, a fellow at the Urban Institute and a co-founder of the Urban-Brookings Tax Policy Center in Washington. “Everything else is getting squeezed.”

        Gradually rising interest rates would have made borrowing more expensive even without additional debt. But the tax cuts passed late last year have created a deeper hole, with the deficit increasing faster than expected. A budget bill approved in February that raised spending by $300 billion over two years will add to the financial pressure.

        The deficit is expected to total nearly $1 trillion next year — the first time it has been that big since 2012, when the economy was still struggling to recover from the financial crisis and interest rates were near zero.

        Annual interest payments on the national debt are expected to triple over the next decade, according to the Congressional Budget Office.

        $915

        billion

        Interest payments on the national debt

        $900

        billion

        600

        300

        $263

        billion

        0

        ’17

        ’18

        ’19

        ’20

        ’21

        ’22

        ’23

        ’24

        ’25

        ’26

        ’27

        ’28

        Interest payments on the national debt

        $900

        billion

        $915

        billion

        600

        300

        $263

        billion

        0

        ’17

        ’18

        ’19

        ’20

        ’21

        ’22

        ’23

        ’24

        ’25

        ’26

        ’27

        ’28

        By The New York Times | Source: Congressional Budget Office

        Deficit hawks have gone silent, even proposing changes that would exacerbate the deficit. House Republicans introduced legislation this month that would make the tax cuts permanent.

        “The issue has just disappeared,” said Senator Mark Warner, a Virginia Democrat. “There’s collective amnesia.”

        The combination, say economists, marks a journey into mostly uncharted financial territory.

        In the past, government borrowing expanded during recessions and waned in recoveries. That countercyclical policy has been a part of the standard Keynesian toolbox to combat downturns since the Great Depression.

        The deficit is soaring now as the economy booms, meaning the stimulus is pro-cyclical. The risk is that the government would have less room to maneuver if the economy slows.

        Debt as a percentage of gross domestic product tends to increase during recessions and fall during recoveries. But the debt is increasing now, even as the economy is growing, because of tax cuts and spending increases.

        Public debt as a share of gross domestic product

        100

        %

        RECESSIONS

        PROJECTED

        80

        60

        40

        20

        0

        ’78

        ’80

        ’90

        ’00

        ’10

        ’20

        ’28

        Public debt as a share of gross domestic product

        100

        %

        80

        60

        40

        20

        RECESSIONS

        PROJECTED

        0

        ’80

        ’90

        ’00

        ’10

        ’20

        ’28

        By The New York Times | Source: Congressional Budget Office

        Aside from wartime or a deep downturn like the 1930s or 2008-9, “this sort of aggressive fiscal stimulus is unprecedented in U.S. history,” said Jeffrey Frankel, an economist at Harvard.

        Pouring gasoline on an already hot economy has resulted in faster growth — the economy expanded at an annualized rate of 4.2 percent in the second quarter. But Mr. Frankel warns that when the economy weakens, the government will find it more difficult to cut taxes or increase spending.

        Lawmakers might, in fact, feel compelled to cut spending as tax revenue falls, further depressing the economy. “There will eventually be another recession, and this increases the chances we will have to slam on the brakes when the car is already going too slowly,” Mr. Frankel said.

        Interest payments will make up 13 percent of the federal budget a decade from now, surpassing spending on Medicaid and defense.

        Expenditures as a share of overall budget

        15

        %

        Net interest

        13.0%

        Defense

        10

        Medicaid

        6.6%

        5

        0

        ’17

        ’18

        ’19

        ’20

        ’21

        ’22

        ’23

        ’24

        ’25

        ’26

        ’27

        ’28

        Expenditures as a share of overall budget

        15

        %

        Net interest

        13.0%

        Defense

        10

        Medicaid

        6.6%

        5

        0

        ’17

        ’18

        ’19

        ’20

        ’21

        ’22

        ’23

        ’24

        ’25

        ’26

        ’27

        ’28

        By The New York Times | Source: Congressional Budget Office

        Finding the money to pay investors who hold government debt will crimp other parts of the budget. In a decade, interest on the debt will eat up 13 percent of government spending, up from 6.6 percent in 2017.

        “By 2020, we will spend more on interest than we do on kids, including education, food stamps and aid to families,” said Marc Goldwein, senior policy director at the Committee for a Responsible Federal Budget, a research and advocacy organization.

        Interest costs already dwarf spending on many popular programs. For example, grants to students from low-income families for college total roughly $30 billion — about one-tenth of what the government will pay in interest this year. Interest payments will overtake Medicaid in 2020 and the Department of Defense budget in 2023.

        What’s more, the heavy burden of interest payments could make it harder for the government to repair aging infrastructure or take on other big new projects.

        Mr. Trump has called for spending $1 trillion on infrastructure, but Congress has not taken up that idea.

        More about the federal debt and the economy
        Trump Administration Mulls a Unilateral Tax Cut for the Rich
        How the Trump Tax Cut Is Helping to Push the Federal Deficit to $1 Trillion

        Until recently, ultralow interest rates, set by the Federal Reserve to support the economy, allowed lawmakers to borrow without fretting too much about the cost of that debt.

        But as the economy has strengthened, the Fed has gradually raised rates, starting in December 2015. The central bank is expected to push rates up again on Wednesday, and more increases are in store.

        “When rates went down to record lows, it allowed the government to take on more debt without paying more interest,” Mr. Goldwein said. “That party is ending.”

        After bottoming out below 2 percent, the yield on the 10-year Treasury note has climbed to over 3 percent recently.

        Yield on 10-year Treasury note

        8

        %

        MONTHLY

        RECESSIONS

        6

        4

        2

        0

        ’98

        ’00

        ’02

        ’04

        ’06

        ’08

        ’10

        ’12

        ’14

        ’16

        ’18

        Yield on 10-year Treasury note

        8

        %

        MONTHLY

        RECESSIONS

        6

        4

        2

        0

        ’98

        ’02

        ’06

        ’10

        ’14

        ’18

        By The New York Times | Source: Reuters

        Since the beginning of the year, the yield on the 10-year Treasury note has risen by more than half a percentage point, to 3.1 percent. The Congressional Budget Office estimates that the yield will climb to 4.2 percent in 2021. Given that the total public debt of the United States stands at nearly $16 trillion, even a small uptick in rates can cost the government billions.

        There’s no guarantee that these forecasts will prove accurate. If the economy weakens, rates might fall or rise only slightly, reducing interest payments. But rates could also overshoot the budget office forecast.

        Some members of Congress want to set the stage for even more red ink. Republicans in the House want to make last year’s tax cuts permanent, instead of letting some of them expire at the end of 2025. That would reduce federal revenue by an additional $631 billion over 10 years, according to the Tax Policy Center.

        The dollar has strengthened even as government borrowing has increased.

        U.S. Dollar index

        140

        MONTHLY

        RECESSIONS

        120

        100

        80

        60

        ’98

        ’00

        ’02

        ’04

        ’06

        ’08

        ’10

        ’12

        ’14

        ’16

        ’18

        U.S. Dollar index

        140

        MONTHLY

        RECESSIONS

        120

        100

        80

        60

        ’98

        ’02

        ’06

        ’10

        ’14

        ’18

        By The New York Times | Source: Reuters

        Deficit hawks have warned for years that a day of reckoning is coming, exposing the United States to the kind of economic crisis that overtook profligate borrowers in the past like Greece or Argentina.

        But most experts say that isn’t likely because the dollar is the world’s reserve currency. As a result, the United States still has plenty of borrowing capacity left because the Fed can print money with fewer consequences than other central banks.

        And interest rates plunged over the last decade, even as the government turned to the market for trillions each year after the recession. That’s because Treasury bonds are still the favored port of international investors in any economic storm.

        “We exported a financial crisis a decade ago, and the world responded by sending us money,” said William G. Gale, a senior fellow at the Brookings Institution.

        But that privileged position has allowed politicians in both parties to avoid politically painful steps like cutting spending or raising taxes.

        That doesn’t mean rapidly rising interest costs and a bigger deficit won’t eventually catch up with us.

        Charles Schultze, chairman of the Council of Economic Advisers in the Carter administration, once summed up the danger of deficits with a metaphor. “It’s not so much a question of the wolf at the door, but termites in the woodwork.”

        Rather than simply splitting along party lines, lawmakers’ attitudes toward the deficit also depend on which party is in power. Republicans pilloried the Obama administration for proposing a large stimulus in the depths of the recession in 2009 and complained about the deficit for years.

        In 2013, Senator Mitch McConnell of Kentucky called the debt and deficit “the transcendent issue of our era.” By 2017, as Senate majority leader, he quickly shepherded the tax cut through Congress.

        Senator James Lankford, an Oklahoma Republican who warned of the deficit’s dangers in the past, nevertheless played down that threat on the Senate floor as the tax billed neared passage.

        “I understand it’s a risk, but I think it’s an appropriate risk to be able to say let’s allow Americans to keep more of their own money to invest in this economy,” he said.

        He also claimed the tax cuts would pay for themselves even as the Congressional Budget Office estimated that they would add $250 billion to the deficit on average from 2019 to 2024.

        In an interview, Mr. Lankford insisted that the jury was still out on whether the tax cuts would generate additional revenue, citing the strong economic growth recently.

        While the Republican about-face has been much more striking, Democrats have adjusted their position, too.

        Mr. Warner, the Virginia Democrat, called last year’s tax bill “the worst piece of legislation we have passed since I arrived in the Senate.” In 2009, however, when Congress passed an $800 billion stimulus bill backed by the Obama administration, he called it “a responsible mix of tax cuts and investments that will create jobs.”

        The difference, Mr. Warner said, was that the economy was near the precipice then.

        “There was virtual unanimity among economists that we needed a stimulus,” he said. “But a $2 trillion tax cut at the end of a business cycle with borrowed money won’t end well.”

        Nelson D. Schwartz has covered economics since 2012. Previously, he wrote about Wall Street and banking, and also served as European economic correspondent in Paris. He joined The Times in 2007 as a feature writer for the Sunday Business section. @NelsonSchwartz

        A version of this article appears in print on , on Page A1 of the New York edition with the headline: What May Soon Exceed Cost of U.S. Military? Interest on U.S. Debt . Order Reprints | Today’s Paper | Subscribe

        Advertisement

        ================================================ FILE: test/test-pages/nytimes-5/expected-metadata.json ================================================ { "title": "The New York Times en Español", "byline": "Tariq Panja", "dir": null, "lang": "es", "excerpt": "Entérate de lo que está pasando en el mundo y de las noticias económicas, de negocios, tecnología, arte, estilos de vida, deporte, ciencia y opiniones. No importa cuáles sean tus intereses: el Times lo cubre con inmejorable calidad, profundidad e independencia.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/nytimes-5/expected.html ================================================

        Highlights

        1. PhotoXi Jinping, el líder de China, arriesgó su prestigio personal cuando su país se postuló para organizar los Juegos de Invierno 2022; hasta ahora el país ha cumplido sus promesas.
          CreditKevin Frayer/Getty Images
        2. Photo
          CreditEllen Surrey

          Ensayo invitado

          El día que renuncié a los Beatles

          Estaba obsesionado con el cuarteto de Liverpool: celebraba sus cumpleaños, leía todo sobre la banda y memoricé todas sus canciones. Cuando me desintoxiqué de ellos descubrí un nuevo mundo musical.

          Por Josh Max

          1. Photo
            CreditErik Carter

            On Tech

            ¿Qué tienen que ver los videojuegos con el metaverso?

            Las compañías tecnológicas creen que los videojuegos son el camino para avanzar más rápido hacia un internet inmersivo. La adquisición de Activision Blizzard por Microsoft es una muestra de esta tendencia.

            Por Shira Ovide

          2. Photo
            CreditTed + Chelsea Cavanaugh para The New York Times

            Skin Deep

            Bebidas con beneficios: ¿de verdad funcionan?

            Hay un nuevo mercado de bebidas que prometen beneficios como la salud intestinal, una mente relajada y piel más brillante. El problema es que ninguno de esos efectos ha sido respaldado científicamente.

            Por Rachel Strugatz

        Opinión

        Más en Opinión ›
        1. Photo
          CreditDanielle Chenette
        2. Photo
          CreditCari Vander Yacht
        3. Photo  
          CreditBianca Bagnarelli
          1. Photo
            CreditKim Raff for The New York Times

        Especial

        Más en Especial ›
        1. Photo 
          CreditPhoto Illustration by Andrew B. Myers for The New York Times

          El desafío Come bien

          Actualiza tus hábitos alimenticios este año, sin necesidad de hacer dieta.

          By Tara Parker-Pope

        2. Photo
          CreditPhoto Illustration by Andrew B. Myers for The New York Times
        3. Photo
          CreditFotoilustraciones de Andrew B. Myers para The New York Times
        4. Photo
          CreditFotoilustración de Andrew B. Myers para The New York Times

        El brote de Coronavirus

        Más en El brote de Coronavirus ›
        1. Photo
          Credit
        2. PhotoLargas filas para hacerse pruebas de coronavirus en Jonesboro, Georgia, este mes. La variante ómicron se identificó a finales de noviembre, por lo que es demasiado pronto para decir cuánto tiempo pueden persistir los síntomas.
          CreditDustin Chambers para The New York Times

          ¿Ómicron puede causar covid prolongada?

          Los científicos dicen que aún es muy pronto para saber si quienes se infectan con la nueva variante tendrán síntomas persistentes. Una infección leve no necesariamente es señal de que hay menos riesgo.

          By Pam Belluck

        3. PhotoA 3-D plaster model of a coronavirus spike protein in the office of Dr. Barney Graham of the Vaccine Research Center of the National Institutes of Health.
          CreditJohnathon Kelso for The New York Times
        4. PhotoUn centro de pruebas de COVID-19 realizadas con saliva en la Universidad de Minnesota, en Mineápolis
          CreditJenn Ackerman para The New York Times
        5. Photo
          CreditCharlie Rubin para The New York Times

        Estados Unidos

        Más en Estados Unidos ›
        1. PhotoUna presentación sobre la Operación Estrella Solitaria, en Weslaco, Texas, el año pasado
          CreditChristopher Lee para The New York Times
        2. PhotoUna multitud se reunió en el National Mall el 6 de enero de 2021, cuando el expresidente Donald Trump cuestionó los resultados de las elecciones de 2020.
          CreditPete Marovich para The New York Times
        3. PhotoNinguna de las más de 729 personas acusadas en relación con los disturbios del Capitolio tiene hasta ahora ninguna conexión con los antifa, según una base de datos de NPR sobre registros de detenciones.
          CreditJason Andrew para The New York Times
        4. PhotoEl expresidente Donald Trump el año pasado. Liz Cheney, representante republicana por Wyoming, ha calificado su lenta respuesta al atentado del 6 de enero como una negligencia en el cumplimiento del deber.
          CreditCooper Neill para The New York Times
        5. PhotoDurante una década, Holmes engañó a inversionistas inteligentes, a cientos de empleados inteligentes, a un comité de figuras ilustres y a los medios de comunicación que estaban ansiosos por ungir a una nueva estrella.
          CreditJenny Hueston

          El auge y la caída de Elizabeth Holmes

          El caso de la fundadora de Theranos podría cambiar el estatus de culto que tienen algunos emprendedores tecnológicos a los que no se les cuestionan sus ambiciosos proyectos ni se les exige que rindan cuentas.

          By David Streitfeld

        ================================================ FILE: test/test-pages/nytimes-5/source.html ================================================ The New York Times en Español - The New York Times
        Skip to contentSkip to site index

        Highlights

        2. Photo
          CreditEllen Surrey

          Ensayo invitado

          El día que renuncié a los Beatles

          Estaba obsesionado con el cuarteto de Liverpool: celebraba sus cumpleaños, leía todo sobre la banda y memoricé todas sus canciones. Cuando me desintoxiqué de ellos descubrí un nuevo mundo musical.

          Por Josh Max

          1. Photo
            CreditErik Carter

            On Tech

            ¿Qué tienen que ver los videojuegos con el metaverso?

            Las compañías tecnológicas creen que los videojuegos son el camino para avanzar más rápido hacia un internet inmersivo. La adquisición de Activision Blizzard por Microsoft es una muestra de esta tendencia.

            Por Shira Ovide

          2. Photo
            CreditTed + Chelsea Cavanaugh para The New York Times

            Skin Deep

            Bebidas con beneficios: ¿de verdad funcionan?

            Hay un nuevo mercado de bebidas que prometen beneficios como la salud intestinal, una mente relajada y piel más brillante. El problema es que ninguno de esos efectos ha sido respaldado científicamente.

            Por Rachel Strugatz

        1. Charles M. Blow

          Photo
          CreditNitashia Johnson para The New York Times

          La muerte cambió mi vida

          Por fuera parecía tener éxito, pero por dentro sentía que me ahogaba. Cuando murió mi hermano, tuve que replantearme todo.

          By Charles M. Blow

        2. EDUCACIÓN FÍSICA

          Photo
          CreditPiotr Redlinski for The New York Times

          Así es como el ejercicio puede controlar nuestra ansiedad

          Los corredores de esquí de fondo son menos propensos a desarrollar trastornos de ansiedad. La buena noticia es que las actividades aeróbicas menos intensas pueden proporcionar beneficios similares.

          By Gretchen Reynolds

        4. EL Desafío Come bien

          Photo
          CreditFotoilustración de Andrew B. Myers para The New York Times

          Siete opciones para alimentar a tu cerebro

          Para el desafío de esta semana, prueba algunos alimentos que se han relacionado con la mejora del estado de ánimo.

          By Tara Parker-Pope

        Opinión

        Más en Opinión ›

        Especial

        Más en Especial ›
        1. Photo 
          CreditPhoto Illustration by Andrew B. Myers for The New York Times

          El desafío Come bien

          Actualiza tus hábitos alimenticios este año, sin necesidad de hacer dieta.

          By Tara Parker-Pope

        3. Photo
          CreditFotoilustraciones de Andrew B. Myers para The New York Times

          ¿Galletas? ¿Papitas? ¿Pizza? Aprende a dominar tus antojos

          Para el desafío Come bien, las técnicas de atención plena, como “surfear los impulsos”, pueden ayudar a frenar el comer demasiado sin prohibir los alimentos favoritos.

          By Tara Parker-Pope

        El brote de Coronavirus

        Más en El brote de Coronavirus ›
        2. PhotoLargas filas para hacerse pruebas de coronavirus en Jonesboro, Georgia, este mes. La variante ómicron se identificó a finales de noviembre, por lo que es demasiado pronto para decir cuánto tiempo pueden persistir los síntomas.
          CreditDustin Chambers para The New York Times

          ¿Ómicron puede causar covid prolongada?

          Los científicos dicen que aún es muy pronto para saber si quienes se infectan con la nueva variante tendrán síntomas persistentes. Una infección leve no necesariamente es señal de que hay menos riesgo.

          By Pam Belluck

        Estados Unidos

        Más en Estados Unidos ›
        5. PhotoDurante una década, Holmes engañó a inversionistas inteligentes, a cientos de empleados inteligentes, a un comité de figuras ilustres y a los medios de comunicación que estaban ansiosos por ungir a una nueva estrella.
          CreditJenny Hueston

          El auge y la caída de Elizabeth Holmes

          El caso de la fundadora de Theranos podría cambiar el estatus de culto que tienen algunos emprendedores tecnológicos a los que no se les cuestionan sus ambiciosos proyectos ni se les exige que rindan cuentas.

          By David Streitfeld

        1. Elogio de la pereza

          Consejo de un rabino, derrame petrolero en Perú, una infección de ómicron en gráficos y más para estar al día.

          Por Elda Cantú

        2. Un ejecutivo de Nike busca el perdón de una familia por un asesinato que cometió en 1965

          Barbara Mack abrazó a Larry Miller después de que este se disculpara por haber matado a su hermano, Edward David White. Pero si fuera 30 años más joven, le dijo, “habría saltado por encima de la mesa para golpearte”.

          Por Jeré Longman y Kevin Draper

          Read in English

        3. EL CUADERNO DE LA CRÍTICA

          Los Ángeles, gloriosa capital del sushi

          Pocos lugares fuera de Japón pueden igualar la variedad, la maestría y la creatividad que se sirve en las innumerables barras de sushi de la ciudad.`

          Por Tejal Rao

          Read in English

        4. ¿Quién entregó a la familia de Ana Frank? Hay una nueva teoría. Es controversial

          Un nuevo libro identifica a un sospechoso de delatar a la familia Frank a la administración nazi. Ahora que expertos e historiadores han tenido tiempo de revisar la investigación, muchos dudan de los métodos y la conclusión.

          Por Nina Siegal

          Read in English

        5. Cuaderno de la crítica

          Cinco estupendas opciones de sushi en Los Ángeles

          En la ciudad abundan los restaurantes donde se despliega estilo y destreza. Estos son algunos de los favoritos de la crítica Tejal Rao.

          Por Tejal Rao

          Read in English

        6. Modern Love

          Tuvimos un divorcio amable y seguimos viviendo en la misma casa

          Cuando el final de un matrimonio significa vivir en pisos separados de la misma residencia.

          Por Jordana Jacobs

          Read in English

        7. Defender Chernóbil en una invasión no vale la pena, dicen algunos ucranianos

          Ucrania ha iniciado una estrategia defensiva para la Zona de Exclusión de Chernóbil, uno de los lugares más radiactivos de la Tierra, que se encuentra en el camino más corto entre Rusia y la capital de Ucrania, Kiev.

          Por Andrew E. Kramer y Tyler Hicks

          Read in English

        8. Tech Fix

          ¿Por qué ha causado tanto alboroto el metaverso?

          Microsoft dijo que pagó 68.700 millones de dólares por una empresa de videojuegos para fortalecer su posición en el metaverso. Veamos qué significa esto en realidad.

          Por Brian X. Chen

          Read in English

        9. Zara Rutherford, la mujer más joven en darle la vuelta al mundo

          Cuando la adolescente aterrizó el jueves, se convirtió en la mujer más joven en circunnavegar el mundo en solitario. “Me alegra poder estar en el mismo lugar por unos meses”, dijo.

          Por Mike Ives

          Read in English

        10. ‘El petróleo va a estar meses en el mar’: un derrame de crudo, provocado por el tsunami, arrasa la costa peruana

          El derrame, que inicialmente se dijo que era de siete galones, resultó ser de 6000 barriles según indicó el Ministerio del Ambiente. El gobierno peruano ha comenzado a pedir ayuda internacional.

          Por Mitra Taj

          Read in English
        ================================================ FILE: test/test-pages/ol/expected-metadata.json ================================================ { "title": "", "byline": null, "dir": null, "excerpt": "AI hasn’t meaningfully changed anything in cybersecurity so far. Deep fake phishing is still rare, L", "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/ol/expected.html ================================================

        1. AI hasn’t meaningfully changed anything in cybersecurity so far. Deep fake phishing is still rare, L

        ================================================ FILE: test/test-pages/ol/source.html ================================================

        1. AI hasn’t meaningfully changed anything in cybersecurity so far. Deep fake phishing is still rare, L

        ================================================ FILE: test/test-pages/parsely-metadata/expected-metadata.json ================================================ { "title": "Some Other Title", "byline": "Jane Doe", "dir": null, "lang": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": "2024-04-20T04:20:00.000Z", "readerable": true } ================================================ FILE: test/test-pages/parsely-metadata/expected.html ================================================

        Test document title

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        ================================================ FILE: test/test-pages/parsely-metadata/source.html ================================================

        Test document title

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        ================================================ FILE: test/test-pages/pixnet/expected-metadata.json ================================================ { "title": "新竹尖石_美樹營地賞楓 (2) @ 史蒂文的家_藍天 :: 痞客邦 PIXNET ::", "byline": "史蒂文的家_藍天 (stevenhgm)", "dir": null, "lang": "zh-TW", "excerpt": "一波波接續性低溫寒流報到 已將新竹尖石鄉後山一帶層層山巒披上嫣紅的彩衣 玉峰道路一路上雲氣山嵐滯留山頭 順路下切蜿蜒道路後不久即抵達來到\"玉峰國小\" \"美樹\"美", "siteName": "史蒂文的家_藍天", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/pixnet/expected.html ================================================

        12-IMG_3886.jpg

        一波波接續性低溫寒流報到 已將新竹尖石鄉後山一帶層層山巒披上嫣紅的彩衣

        玉峰道路一路上雲氣山嵐滯留山頭 順路下切蜿蜒道路後不久即抵達來到"玉峰國小"

        "美樹"美如其名偌大楓香樹早已呈現金黃 泛紅色彩也是愛攝人仕所喜愛造訪之地

        第二次造訪美樹發現到營區變了 印象中以前生冷招牌換成了原木招牌可謂匠心獨運

        燻黑原木加上金黃色醒目字體 加上了貓頭鷹原木創作也充分發揮了裝飾藝術功力

        營區內除了露營、民宿、餐飲、賞楓項目多了許多原木飾品更有畫龍點睛加乘效果

        30-IMG_4228.jpg

        廣受歡迎的美樹營地有個很大特色就是楓紅時期楓香樹由綠轉黃、轉紅到楓紅層層

        一來到"美樹"馬上眼睛為之一亮　也會深深地為那多種顏色多層次渲染之下楓紅而迷惑

        不同格調就是從入口處這塊招牌第一眼開始 木頭招牌 、貓頭鷹裝飾品勾勒出美樹的風格

        31-IMG_4231.jpg

        每年12月向來是攝影班外拍的絕佳場所之一 楓紅期間入園費$50元

        園區給愛攝一族淨空場景而不是散搭帳蓬之下反而影響拍照畫面與構圖取景

        露營的話則須待中午過後再進場搭帳的彈性做法個人也相當支持這樣的權宜之計

        P1610088.jpg

        來到現場已是落葉飄飄堆疊滿地　不時隨著風吹雨襲而葉落垂地

        P1610069.jpg

        不忍踩過剛剛掉落的樹葉 沿著前人足跡踏痕輕踩而行

        雖然只是一廂情願的想法 終究還是不可避免地將會化為塵土

        02-P1610080.jpg

        葉落繽紛顯得幾分蕭瑟氣息 空氣中可以嗅得出來依然瀰漫著濕寒水氣

        偶而還會飄下來一些霧氣水滴 不時張望尋找最佳楓葉主題

        04-P1610087.jpg

        外拍的攝影班學員一堆早已不時穿梭其間

        各自努力地找尋自認為最好的拍攝角度

        05-P1610099.jpg

        P1610095.jpg

        13-IMG_3891.jpg

        15-IMG_3906.jpg

        "水槽"上面的這幾隻彩繪版貓頭鷹也太可愛了

        同樣的造型加上不同色彩宛如賦予不同的生命力一般　cool!

        16-IMG_3916.jpg

        雨水洗塵後的枝頭固然掉落些葉片是否也洗去塵勞憂傷

        17-IMG_3919.jpg

        06-IMG_3853.jpg

        喜歡拍照的不論是平面掃描、天空搜尋、地上地毯式搜索

        有如小說偵探一般 不放過蛛絲馬跡地用力尋尋覓覓找尋最美角度

        07-P1610104.jpg

        08-IMG_3862.jpg

        原本這周是由小朱團長早在一年前就跟"簍信"預定下來的場子

        早上從台北出門之際還是小雨不斷細雨紛飛來到此地雖雨已停

        但多日來的雨勢不斷已有部分區域水漬成攤並不適合落置帳篷

        這個季節正是"控溪吊橋"一帶的楓紅變葉時刻 先行走一趟秀巒賞景

        18-P1610141.jpg

        午後從"秀巒"回到美樹之際已經全數撤退只剩下我們三車留下來

        唯有"離開地球表面"睡車上的才可以不受到地上泥濘而影響

        19-IMG_3933.jpg

        14-P1610134.jpg

        午後山嵐興起雲氣遊蕩盤旋在對岸山頭　人潮來來去去似乎也沒有減少

        44-P1610283.jpg

        美樹民宿有開設餐廳 室內簡單佈置提供伙食餐飲

        P1610212.jpg

        這兩間是民宿房間 跟著民宿主人"簍信"聊起來還提到日後將改變成兩層木屋

        一樓則是咖啡飲料/賣店提供訪客來賓有個落腳席座之地 二樓才會是民宿房間

        心中有了計畫想法才會有日後的夢想藍圖 相信將會改變得更好的民宿露營環境

        P1610219.jpg

        民宿前這一大區楓香林為土質營位 大致區分前、後兩個營區

        前面這一區約可搭上十二帳/車/廳 後面那區也大約4~5帳/車/廳

        正前方小木屋即是衛浴區 男女分別以左右兩側分開(燒材鍋爐)

        10-P1610114.jpg

        營區水電方便 水槽也很有特色

        22-P1610245.jpg

        這次選擇左側地勢高些以防午夜下雨泥濘

        20-P1610238.jpg

        "野馬"特地帶來了冬至應景食材ㄜ---湯圓

        這家還是最近被評比第一名氣的湯圓專賣店

        21-P1610241.jpg

        向來對於湯圓是敬謝不敏 沒想到是出乎意料之外的好吃 沒話說!

        24-IMG_4113.jpg

        喜歡原住民朋友的坦率、真誠　要將民宿營地經營的有聲有色並非容易之事

        午茶時間與"簍信"閒聊分享著他的觀點理念之時很支持對於環境應有生態保護

        環保維護是人人有責 勿以善小而不為不計涓滴之水才可匯集成河　

        32-IMG_4248.jpg

        25-IMG_4152.jpg

        入夜前雨絲終於漸漸緩和下來　雖然氣溫很低卻沒感受到寒冷的跡象

        是山谷中少了寒氣還是美樹營區裡的人熱情洋溢暖化了不少寒意

        IMG_4158.jpg

        聖誕前夕裝點些聖誕飾品 感受一下節慶的氛圍

        26-P1610261.jpg

        晚餐準備了砂鍋魚頭

        46-1021221美樹露營.jpg

        "蒯嫂"還特地準備著羊肩排、鹹豬肉、柳葉魚...哇！這哩澎湃哩...

        　"永老爺"早已備妥了好酒為遠自台南來的蒯兄嫂敬一杯囉

        感謝蒯嫂精心準備的好料理　食指大動好菜色感恩ㄟ！

        27-IMG_4173.jpg

        吃得快精光之際...才想到忘了拍合照...(哇哩咧 ^&*()

        28-IMG_4178.jpg

        29-IMG_4188.jpg

        隔日睡到很晚才起床 不用拍日出晨光的營地對我來說都是個幸福的睡眠

        哪怕是葉落飄零落滿地還是睡夢周公召見而去　起床的事~差點都忘記了

        IMG_4205.jpg

        昨天細雨紛飛依然打落了不少落葉中間這株整個都快變成枯枝了

        昨天依稀凋零稀疏的楓葉殘留今兒個完全不復存在（上周是最美的代名詞）

        33-IMG_4255.jpg

        上回來得太早沒能見到楓葉泛紅 這次晚了一周已陸續落葉也無從比對楓葉差異性　

        另一種角度看不論青楓、金黃葉紅的楓香、葉落飄零秋滿霜、落葉枯枝的蕭瑟

        只要心中自認為是最美最浪漫的一刻 都是美的表徵也是最美的時分

        34-P1610269.jpg

        早起的"蒯嫂"已經備好熱騰騰中式稀飯、包子、蔬果　頓時~有幸福的感覺

        35-IMG_4303.jpg

        星期天早上趁著攝影團還沒入場先來人物場景特寫

        野馬家兩張新"座椅"就當作是試坐囉！拍謝哩

        38-IMG_4330.jpg

        P1610279.jpg

        難得有此無人美景在楓樹下的聖誕氛圍也一定要來一張才行

        37-IMG_4323.jpg

        三家合照（Hero也一定要入鏡的）

        40-IMG_4342.jpg

        接著攝影團入場帶隊老師請求借個時間也來讓學員練習楓樹下的聖誕飾品

        此時剛好也遇到早在FB社團相互回應卻頭一次謀面的Mr."大雄"真是幸會了

        42-IMG_4382.jpg

        接近中午時分陽光漸露　藍天帷幕再次嶄露頭角 ~ 久違了!

        期盼下的天空終於放晴 沒有缺席的藍天還是準時赴約如期出席

        41-IMG_4366.jpg

        這兩天肉肉(Hero)天雨濕滑無法自由奔跑都快悶壞了

        天晴後"蒯嫂"帶著散步遊園也好解解悶

        43-IMG_4383.jpg

        收拾好裝備準備離開營地 亮麗的天空鮮明對比下的楓樹林又讓人覺得有點捨不得離開

        道別了"美樹營地"準備前往而行"石磊國小"一個很生疏的小學座落在這深山部落裡

        北橫"石磊部落" 一個從未踏入的陌生之地因為露營之故否則畢生大概也不會路過

        三位大叔同行準備走著這段遙遠的路段 下次找機會再來重溫舊夢了.......

        美樹營地 資訊

        聯絡電話：03-584-7231　 行動: 0937-141993
        林錦武 (泰雅族名: 摟信)
        營地地址：新竹縣尖石鄉玉峰村6鄰20號

        每帳$600 兩間衛浴使用燒材鍋爐/ 兩間全天瓦斯 廁所蹲式X 3

        楓紅期間須過中午才可搭帳　水電便利

        GPS: N24 39 16.4 E121 18 19.5

        如果您喜歡"史蒂文的家"圖文分享 邀請您到 FB 粉絲團 按個"讚"!

        內文有不定期的更新旅遊、露營圖文訊息 謝謝!

        ================================================ FILE: test/test-pages/pixnet/source.html ================================================ 新竹尖石_美樹營地賞楓 (2) @ 史蒂文的家_藍天 :: 痞客邦 PIXNET ::
        依據 Google 關鍵字搜尋結果
          相關文章

          12-IMG_3886.jpg

          一波波接續性低溫寒流報到 已將新竹尖石鄉後山一帶層層山巒披上嫣紅的彩衣

          玉峰道路一路上雲氣山嵐滯留山頭 順路下切蜿蜒道路後不久即抵達來到"玉峰國小"

          "美樹"美如其名偌大楓香樹早已呈現金黃 泛紅色彩也是愛攝人仕所喜愛造訪之地

          第二次造訪美樹發現到營區變了 印象中以前生冷招牌換成了原木招牌可謂匠心獨運

          燻黑原木加上金黃色醒目字體 加上了貓頭鷹原木創作也充分發揮了裝飾藝術功力

          營區內除了露營、民宿、餐飲、賞楓項目多了許多原木飾品更有畫龍點睛加乘效果


          30-IMG_4228.jpg

          廣受歡迎的美樹營地有個很大特色就是楓紅時期楓香樹由綠轉黃、轉紅到楓紅層層

          一來到"美樹"馬上眼睛為之一亮　也會深深地為那多種顏色多層次渲染之下楓紅而迷惑

          不同格調就是從入口處這塊招牌第一眼開始 木頭招牌 、貓頭鷹裝飾品勾勒出美樹的風格

          31-IMG_4231.jpg

          每年12月向來是攝影班外拍的絕佳場所之一 楓紅期間入園費$50元

          園區給愛攝一族淨空場景而不是散搭帳蓬之下反而影響拍照畫面與構圖取景

          露營的話則須待中午過後再進場搭帳的彈性做法個人也相當支持這樣的權宜之計

          P1610088.jpg

          來到現場已是落葉飄飄堆疊滿地　不時隨著風吹雨襲而葉落垂地

          P1610069.jpg

          不忍踩過剛剛掉落的樹葉 沿著前人足跡踏痕輕踩而行

          雖然只是一廂情願的想法 終究還是不可避免地將會化為塵土

          02-P1610080.jpg

          葉落繽紛顯得幾分蕭瑟氣息 空氣中可以嗅得出來依然瀰漫著濕寒水氣

          偶而還會飄下來一些霧氣水滴 不時張望尋找最佳楓葉主題

          04-P1610087.jpg

          外拍的攝影班學員一堆早已不時穿梭其間

          各自努力地找尋自認為最好的拍攝角度

          05-P1610099.jpg

          P1610095.jpg

          13-IMG_3891.jpg

          15-IMG_3906.jpg

          "水槽"上面的這幾隻彩繪版貓頭鷹也太可愛了

          同樣的造型加上不同色彩宛如賦予不同的生命力一般　cool!

          16-IMG_3916.jpg

          雨水洗塵後的枝頭固然掉落些葉片是否也洗去塵勞憂傷

          17-IMG_3919.jpg

          06-IMG_3853.jpg

          喜歡拍照的不論是平面掃描、天空搜尋、地上地毯式搜索

          有如小說偵探一般 不放過蛛絲馬跡地用力尋尋覓覓找尋最美角度

          07-P1610104.jpg

          08-IMG_3862.jpg

          原本這周是由小朱團長早在一年前就跟"簍信"預定下來的場子

          早上從台北出門之際還是小雨不斷細雨紛飛來到此地雖雨已停

          但多日來的雨勢不斷已有部分區域水漬成攤並不適合落置帳篷

          這個季節正是"控溪吊橋"一帶的楓紅變葉時刻 先行走一趟秀巒賞景

          18-P1610141.jpg

          午後從"秀巒"回到美樹之際已經全數撤退只剩下我們三車留下來

          唯有"離開地球表面"睡車上的才可以不受到地上泥濘而影響

          19-IMG_3933.jpg

          14-P1610134.jpg

          午後山嵐興起雲氣遊蕩盤旋在對岸山頭　人潮來來去去似乎也沒有減少

          44-P1610283.jpg

          美樹民宿有開設餐廳 室內簡單佈置提供伙食餐飲

          P1610212.jpg

          這兩間是民宿房間 跟著民宿主人"簍信"聊起來還提到日後將改變成兩層木屋

          一樓則是咖啡飲料/賣店提供訪客來賓有個落腳席座之地 二樓才會是民宿房間

          心中有了計畫想法才會有日後的夢想藍圖 相信將會改變得更好的民宿露營環境

          P1610219.jpg

          民宿前這一大區楓香林為土質營位 大致區分前、後兩個營區

          前面這一區約可搭上十二帳/車/廳 後面那區也大約4~5帳/車/廳

          正前方小木屋即是衛浴區 男女分別以左右兩側分開(燒材鍋爐)

          10-P1610114.jpg

          營區水電方便 水槽也很有特色

          22-P1610245.jpg

          這次選擇左側地勢高些以防午夜下雨泥濘

          20-P1610238.jpg

          "野馬"特地帶來了冬至應景食材ㄜ---湯圓

          這家還是最近被評比第一名氣的湯圓專賣店

          21-P1610241.jpg

          向來對於湯圓是敬謝不敏 沒想到是出乎意料之外的好吃 沒話說!

          24-IMG_4113.jpg

          喜歡原住民朋友的坦率、真誠　要將民宿營地經營的有聲有色並非容易之事

          午茶時間與"簍信"閒聊分享著他的觀點理念之時很支持對於環境應有生態保護

          環保維護是人人有責 勿以善小而不為不計涓滴之水才可匯集成河　

          32-IMG_4248.jpg

          25-IMG_4152.jpg

          入夜前雨絲終於漸漸緩和下來　雖然氣溫很低卻沒感受到寒冷的跡象

          是山谷中少了寒氣還是美樹營區裡的人熱情洋溢暖化了不少寒意


          IMG_4158.jpg

          聖誕前夕裝點些聖誕飾品 感受一下節慶的氛圍

          26-P1610261.jpg

          晚餐準備了砂鍋魚頭

          46-1021221美樹露營.jpg

          "蒯嫂"還特地準備著羊肩排、鹹豬肉、柳葉魚...哇！這哩澎湃哩...

          　"永老爺"早已備妥了好酒為遠自台南來的蒯兄嫂敬一杯囉

          感謝蒯嫂精心準備的好料理　食指大動好菜色感恩ㄟ！

          27-IMG_4173.jpg

          吃得快精光之際...才想到忘了拍合照...(哇哩咧 ^&*()

          28-IMG_4178.jpg

          29-IMG_4188.jpg

          隔日睡到很晚才起床 不用拍日出晨光的營地對我來說都是個幸福的睡眠

          哪怕是葉落飄零落滿地還是睡夢周公召見而去　起床的事~差點都忘記了

          IMG_4205.jpg

          昨天細雨紛飛依然打落了不少落葉中間這株整個都快變成枯枝了

          昨天依稀凋零稀疏的楓葉殘留今兒個完全不復存在（上周是最美的代名詞）

          33-IMG_4255.jpg

          上回來得太早沒能見到楓葉泛紅 這次晚了一周已陸續落葉也無從比對楓葉差異性　

          另一種角度看不論青楓、金黃葉紅的楓香、葉落飄零秋滿霜、落葉枯枝的蕭瑟

          只要心中自認為是最美最浪漫的一刻 都是美的表徵也是最美的時分

          34-P1610269.jpg

          早起的"蒯嫂"已經備好熱騰騰中式稀飯、包子、蔬果　頓時~有幸福的感覺

          35-IMG_4303.jpg

          星期天早上趁著攝影團還沒入場先來人物場景特寫

          野馬家兩張新"座椅"就當作是試坐囉！拍謝哩

          38-IMG_4330.jpg

          P1610279.jpg

          難得有此無人美景在楓樹下的聖誕氛圍也一定要來一張才行

          37-IMG_4323.jpg

          三家合照（Hero也一定要入鏡的）

          40-IMG_4342.jpg

          接著攝影團入場帶隊老師請求借個時間也來讓學員練習楓樹下的聖誕飾品

          此時剛好也遇到早在FB社團相互回應卻頭一次謀面的Mr."大雄"真是幸會了

          42-IMG_4382.jpg

          接近中午時分陽光漸露　藍天帷幕再次嶄露頭角 ~ 久違了!

          期盼下的天空終於放晴 沒有缺席的藍天還是準時赴約如期出席

          41-IMG_4366.jpg

          這兩天肉肉(Hero)天雨濕滑無法自由奔跑都快悶壞了

          天晴後"蒯嫂"帶著散步遊園也好解解悶

          43-IMG_4383.jpg

          收拾好裝備準備離開營地 亮麗的天空鮮明對比下的楓樹林又讓人覺得有點捨不得離開

          道別了"美樹營地"準備前往而行"石磊國小"一個很生疏的小學座落在這深山部落裡

          北橫"石磊部落" 一個從未踏入的陌生之地因為露營之故否則畢生大概也不會路過

          三位大叔同行準備走著這段遙遠的路段 下次找機會再來重溫舊夢了.......

          美樹營地 資訊

          聯絡電話：03-584-7231　 行動: 0937-141993
          林錦武 (泰雅族名: 摟信)
          營地地址：新竹縣尖石鄉玉峰村6鄰20號

          每帳$600 兩間衛浴使用燒材鍋爐/ 兩間全天瓦斯 廁所蹲式X 3

          楓紅期間須過中午才可搭帳　水電便利

          GPS: N24 39 16.4 E121 18 19.5

          如果您喜歡"史蒂文的家"圖文分享 邀請您到 FB 粉絲團 按個"讚"!

          內文有不定期的更新旅遊、露營圖文訊息 謝謝!

          , , , ,
          上一篇文章與下一篇文章
          您可能會有興趣的文章

          史蒂文的家_藍天 發表在 痞客邦 PIXNET 留言(42) 人氣(23925)

          產生短網址 E-mail轉寄

          ▲top

          留言列表 (42)

          發表留言
          • #2 茜媽咪 茜媽咪 於 2013/12/28 10:42
          • 茜媽咪

          • 嚮往大自然....再濕冷的天依舊衝衝衝
            每每分享藍天大哥捕捉的鏡頭
            總是特別的吸引目光....真得好美
            對了....我家女兒說....伯伯的露營餐點
            看起來好豐盛喔
          • 茜媽咪晚安：
            您過獎了,我們不是專業攝影師只是喜歡拍照而已,
            雨天過後的楓葉蕭瑟感變厚重了,也顯得很鮮明色澤而已！
            這次還是南部上來的朋友帶來豐盛的晚餐,真是不好意思ㄋ！
            （拍謝！外出帶著平板電腦不容易回應..才剛剛回家回覆）

            史蒂文的家_藍天 於 2014/01/01 20:37 回覆

          • #3 訪客 於 2013/12/28 20:34
          • 訪客

          • 藍天哥

            您的文筆真是讚

            大雄
          • 大雄　晚安：
            毋影啦～只是有感而發的抒發一點所見所聞而已！

            史蒂文的家_藍天 於 2014/01/01 20:39 回覆

          • #4 linghua linghua 於 2013/12/28 21:29
          • linghua

          • 哇！好厚的紅毯 趴在地上取景辛苦了！
            近年天氣消笑 忽冷忽熱
            外拍能遇到最美的藍天和紅葉 可遇不可求....
            桌上的砂鍋魚頭看起來超讚
            冷冷的寒冬享用 一定過癮極了！
          • 哈～我的相機可以翻轉螢幕ㄜ,當初也有考慮到這場景的辛苦...
            所以螢幕可以折疊就輕鬆多囉！
            楓紅總是不等人的,我們這一周跟上周相比較的話就差很大,
            選對時間才是做重要！　天寒之際吃火鍋最方便,也是偷懶...

            史蒂文的家_藍天 於 2014/01/01 20:59 回覆

          • #5 青草 青草 於 2013/12/28 22:50
          • 青草

          • 大哥晚安~
            今天路過美樹營地,真的很美!
            可惜現在楓已落的差不多了!!
            好深山那內的地方,我們由秀巒下玉峰
            再由北橫三光回家,一路的山景秀麗!!
          • 青草晚安：
            上週幾天下來又是風又是雨,
            "花落知多少..."就算差一周也
            差很大呢！明年算準時間再訪
            就一定很棒的！

            史蒂文的家_藍天 於 2014/01/01 20:56 回覆

          • #6 阿蒯 阿蒯 於 2013/12/29 09:50
          • 阿蒯

          • 大推
            不是因為文中的很多”蒯嫂”
            回家之後發現我的”恐山症”竟然好了八成
            那段石磊產業道路 心理暗唸”阿彌陀佛”不只百遍除了感謝
            我詞窮了
            你們與野馬伉儷的友誼讓我們在陰雨寒冷中覺得溫暖
            這是一次另人感動的旅程
            有你們真好！
          • 蒯兄嫂晚安：
            這趟路讓您辛苦了,雖然天氣不佳依然如期赴約,
            讓蒯嫂處處打點得得這麼好,真有點過意不去...
            感恩！相信"之前"的車況給受驚擾了,如今應該
            是恢復信心,有藍天在的話,出門都是安全無虞的！

            史蒂文的家_藍天 於 2014/01/01 20:54 回覆

          • #7 甄妮佛 甄妮佛 於 2013/12/29 16:01
          • 甄妮佛

          • 山嵐 楓樹 落葉 ... 這地方太美了!! 太美了!!
          • 甄妮佛　晚安：
            如果有露營的朋友一定也聽過有這個地方,
            時令季節一到就是楓紅滿地...尖石秀巒一帶
            很有秋天的氣息哦....

            史蒂文的家_藍天 於 2014/01/01 20:48 回覆

          • #8 Cora Cora 寄信給 Cora 於 2013/12/29 20:27
          • Cora

          • 雨霧飄渺中的楓黃楓紅另有一番風韻
          • Cora晚安：
            幾天下來楓葉經過雨後洗滌變得很鮮明,
            當然也帶走許多落葉..蕭瑟唯美另一種風情！

            史蒂文的家_藍天 於 2014/01/01 20:46 回覆

          • #9 tinned tinned 於 2013/12/30 16:49
          • tinned

          • 真是拍的漂亮 我是北七強
          • 北七強　晚安：
            謝謝不吝讚美,其實我們只是喜歡拍而已,畢竟跟專業攝影還是有段距離,
            簍信能有有您這些朋友真是太幸福了...,美樹愈來愈進步您是功不可沒...
            (哈～這個名稱取得...似乎很有玄機呢？)

            史蒂文的家_藍天 於 2014/01/01 20:44 回覆

          • #10 莉莉夫子 莉莉夫子 於 2013/12/30 20:37
          • 莉莉夫子

          • 藍天大哥:晚安~
            哇~睡車上.機動性很強咧~
            今年的美樹 好美啊~
          • LiLi夫子晚安：
            是呀！改裝成車中床離開"地球表面",力克
            潮溼地表(懶得收帳篷才是事實）..
            今年的"美樹"楓紅狀況很優,我們去的上一周
            更是最大值,看到ＦＢ有朋友分享..美的驚豔！

            史蒂文的家_藍天 於 2014/01/01 20:25 回覆

          • #11 黑傑克 黑傑克 於 2013/12/31 22:50
          • 黑傑克

          • 在藍天大的掌鏡下，美樹果然是仙境呀~
          • 黑傑克晚安:
            拍謝!出門五天才剛剛入門,外出使用平板電腦
            實在很不方便中文輸入跟回應....
            您客氣了,我們只是很一般的而已,看了您的
            瑞士之行...才是仙境,會流口水哦!

            史蒂文的家_藍天 於 2014/01/01 20:17 回覆

          • 悄悄話
          • #13 飛兒 飛兒 於 2014/01/02 14:31
          • 飛兒

          • 那年 我不是才不約而同到美樹賞楓
            好快 又過了一年
            山嵐在遠方飄移 讓秋楓更有意境
            美樹的楓香 不管怎麼拍都很美
            石磊部落 我在原民台看過介紹
            是一個以種植有機蔬果為主的部落
            也是一個很美的地方 我還沒去過
            期待藍天大哥的分享
          • 飛兒晚安:
            說的也是,那年真的是不約而同,
            只差沒真的碰面而已..如今又已
            倏逾兩年了...
            回程走"石磊道路"再轉回北橫巴崚,
            切回三峽回台北,路況好遠喲...

            史蒂文的家_藍天 於 2014/01/02 23:36 回覆

          • #14 王昆 王昆 於 2014/01/02 15:53
          • 王昆

          • 欣賞好圖文唷~
            安安~工作愉快~~順心~
          • 謝謝賞圖!
            祝福~順遂

            史蒂文的家_藍天 於 2014/01/13 23:05 回覆

          • #15 大小姐 大小姐 於 2014/01/02 17:44
          • 大小姐

          • 真是讓人羨慕的生活喔
          • 大小姐 晚安:
            露營是另外一種戶外生活,您的餐飲美食生活
            也是不遑多讓ㄜ...

            史蒂文的家_藍天 於 2014/01/02 23:30 回覆

          • #16 王昆 王昆 於 2014/01/02 20:32
          • 王昆

          • 賞圖文唷~美唷!! 善知識晚安安

            祝您馬年行大運馬上賺大錢唷~
          • 也祝您 新春愉快,新的一年心想事成,萬事如意

            史蒂文的家_藍天 於 2014/01/02 23:59 回覆

          • #17 光頭 光頭 於 2014/01/02 21:52
          • 光頭

          • 楓樹 落葉~這地方太美~拍照真美
          • 光頭 晚安:
            這格地點的楓香泛紅之際是最美的時刻,
            人在現場看更是會感動的地方...

            史蒂文的家_藍天 於 2014/01/02 23:32 回覆

          • 悄悄話
          • #19 Sheila Teng Sheila Teng 於 2014/01/03 19:41
          • Sheila Teng

          • 台灣能找到這麼厚實的楓葉地毯...不容易呀!!
            此時若想低拍就會羨慕起那些螢幕可翻轉的相機了~

            控溪吊橋那邊我拍過, 風景真的很棒!
            美樹名字取得簡單乾脆...
            這個地方記下來了, 不必去石門水庫人擠人
            這裡的楓葉更有看頭哩!!!
          • Sheila晚安:
            來到秀巒軍艦岩、控溪吊橋拍秋楓的話,
            那就得順道去"美樹"拍楓香林才不需此行,
            一個簡單的營地卻是愛攝人士每年必訪之地,
            攝影老師總是帶著學員來此地外拍練習,
            要是妳來拍攝的話,必然精彩可期,"另眼看待"ㄜ!

            史蒂文的家_藍天 於 2014/01/03 22:07 回覆

          • #20 阿麗瑪 阿麗瑪 於 2014/01/03 22:44
          • 阿麗瑪

          • 先來說個老梗～落花並非無情物，化作春泥更護花！♥
            落葉鋪成的地毯真的感覺粉溫馨吼？
            哦～你們南來北往大會師噢？
            美樹營地好美呢！偶們一心一意直奔司馬庫斯所以沒有近來賞景．．．
            伙食好讚噢！偶們四餐有兩餐試吃泡麵，早餐是吃隔夜飯泡水的硬硬不濃稠稀飯．．．☹
            恭喜你們收穫滿行曩♣★
            噢對！還有怎麼睡車上呢？露營車嗎？偶們去年元旦去國境之南找不到旅館也是當了車床族～睡車上．．．好累！︿＿＊
          • 阿麗瑪晚安:
            這個季節的楓香林剛好泛紅,剛好連日來雨勢不斷變成
            落葉成塚,否則滿滿色彩的楓樹是最美的! 阿蒯之家特地
            從南部北上一起同露,蒯嫂手藝很棒我們才有口福哩!
            我們是休旅車,經由專門店家改裝成"活動式車床",拆卸快速
            只需要五分鐘就可以搞定,長度180公分(含海棉墊)這樣睡眠
            才有品質,一覺到天亮甚至會睡過頭...

            史蒂文的家_藍天 於 2014/01/03 23:11 回覆

          • 悄悄話
          • #23 lily lily 於 2014/01/09 15:20
          • lily

          • 第一張相片的貓頭鷹實在是太吸睛了~~
            這裡賞楓露營真是好地方
            出去露營能吃到如此豐盛
            真是超級羨慕ㄋ
          • Lily　晚安：
            美樹營地招牌上分別佈置了幾隻貓頭鷹,
            顏色鮮豔也很可愛,也成為愛攝人士的鏡頭
            追逐焦點,有機會可以去看一看！

            史蒂文的家_藍天 於 2014/01/09 21:10 回覆

          • #24 tinned tinned 於 2014/01/27 15:28
          • tinned

          • 北七強的由來ㄚ
            就很簡單ㄚ 畫圖的時候很單純 很少去想最後的結果

            北七 還挺不錯的ㄚ 可能是指ㄧ個心境問題喔
          • 簍信,有緣認識你們這些朋友真是幸運了!
            "北七" 的取名太有意思了~~
            (拍謝, 今早上才見到有漏回覆的,金害ㄟ)

            史蒂文的家_藍天 於 2014/02/10 10:22 回覆

          • #26 訪客 於 2015/11/30 12:45
          • 訪客

          • 真的拍得好棒好溫馨~~
          • 謝謝賞圖/文!

            史蒂文的家_藍天 於 2015/11/30 17:40 回覆

          • #29 米可 米可 於 2016/07/28 15:07
          • 米可

          • 這太美了！好喜歡，我從沒看過，希望有一天可以去看看走一趟！
          • 這裡是新竹縣秀巒鄉很著名的攝影景點,
            如果不是露營的話也可以趁著早上開放時間拍照,
            付入園會即可入內盡情拍照...
            PS:每年楓紅時間"大約"是12月中...左右,
            實際還得視當地氣溫而定

            史蒂文的家_藍天 於 2016/08/02 10:37 回覆

          • #32 Quby Quby 於 2016/08/31 21:15
          • Quby

          • 再過不久又進入秋天了
            楓葉照令人很期待呢
            而且到時候的天氣適合露營
            不會很濕熱也不會太冷
          • 入秋之後寒流來襲就容易讓秋楓變色,
            秋高氣爽野正是露營的好時節

            史蒂文的家_藍天 於 2016/09/08 18:37 回覆

          • #34 木博士 木博士 於 2016/09/19 19:05
          • 木博士

          • 很美~美術營是帶小朋友去玩的好地方!!
          • 美樹營地有奇迷人之處,春夏秋冬各有其特色,
            小朋友玩耍也很安全的!

            史蒂文的家_藍天 於 2016/09/23 09:38 回覆

          • #37 Jenny10113 Jenny10113 於 2016/09/29 19:46
          • Jenny10113

          • 好夢幻喔!!!!!
          • 很美的楓香營地,楓紅時節也是熱門的拍照景點

            史蒂文的家_藍天 於 2016/10/04 09:46 回覆

          • #39 廣州十大旅遊景點 廣州十大旅遊景點 於 2016/10/11 11:02
          • 廣州十大旅遊景點

          • 這篇寫的很好
            歡迎回訪喔
          • 謝謝你!

            史蒂文的家_藍天 於 2016/10/12 14:25 回覆

          • #40 tw27003265 tw27003265 於 2016/10/17 19:47
          • tw27003265

          • 又是看秋景的季節

          您尚未登入，將以訪客身份留言。亦可登入留言

          登入可留悄悄話
          其他選項
          ================================================ FILE: test/test-pages/qq/expected-metadata.json ================================================ { "title": "DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶_科技_腾讯网", "byline": null, "dir": null, "excerpt": "DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/qq/expected.html ================================================

          TNW中文站2016年10月14日07:17

          转播到腾讯微博

          DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶

          TNW中文站 10月14日报道

          谷歌(微博) 在2014年收购的人工智能公司DeepMind开发出一款能够用自己的记忆学习新知识并利用这些知识来回答问题的计算机。

          这款产品具有极其重要的意义，因为这意味着未来的人工智能技术可能不需要人类来教它就能回答人类提出的问题。

          DeepMind表示，这款名为DNC（可微神经计算机）的AI模型可以接受家谱和伦敦地铁网络地图这样的信息，还可以回答与那些数据结构中的不同项目之间的关系有关的复杂问题。

          例如，它可以回答“从邦德街开始，沿着中央线坐一站，环线坐四站，然后转朱比利线坐两站，你会到达哪个站？”这样的问题。

          DeepMind称，DNC还可以帮你规划从沼泽门到皮卡迪利广场的最佳路线。

          同样，它还可以理解和回答某个大家族中的成员之间的关系这样的复杂问题，比如“张三的大舅是谁？”。

          DNC建立在神经网络的概念之上，神经网络可以模拟人类思想活动的方式。这种AI技术很适合与机器习得配套使用。

          DeepMind的AlphaGo AI能够打败围棋冠军也跟这些神经网络有很大关系。但是AlphaGo必须进行训练才行，开发人员向AlphaGo提供了历史对弈中的大约3000万记录。让人工智能技术具备通过记忆学习的能力，就可以让它独自完成更复杂的任务。

          DeepMind希望DNC可以推动计算行业实现更多突破。DeepMind已将其研究结果发表在科学刊物《自然》（Nature）上。（编译/林靖东）

          精彩视频推荐

          转播到腾讯微博

          【美国The Next Web作品的中文相关权益归腾讯公司独家所有。未经授权，不得转载、摘编等。】

          [责任编辑：alonliu]

          您认为这篇文章与"新一网（08008.HK）"相关度高吗？

          ================================================ FILE: test/test-pages/qq/source.html ================================================ DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶_科技_腾讯网 无障碍说明
          浏览器更新提醒电脑登录微信无需扫码，浏览QQ空间提速 5 倍立即更新×
          腾讯科技 科技 > 要闻 > THE NEXT WEB > 正文
          10

          DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶

          TNW中文站2016年10月14日07:17
          转播到腾讯微博
          DeepMind新电脑已可利用记忆自学 人工智能迈上新台阶

          TNW中文站 10月14日报道

          谷歌(微博) 在2014年收购的人工智能公司DeepMind开发出一款能够用自己的记忆学习新知识并利用这些知识来回答问题的计算机。

          这款产品具有极其重要的意义，因为这意味着未来的人工智能技术可能不需要人类来教它就能回答人类提出的问题。

          DeepMind表示，这款名为DNC（可微神经计算机）的AI模型可以接受家谱和伦敦地铁网络地图这样的信息，还可以回答与那些数据结构中的不同项目之间的关系有关的复杂问题。

          例如，它可以回答“从邦德街开始，沿着中央线坐一站，环线坐四站，然后转朱比利线坐两站，你会到达哪个站？”这样的问题。

          DeepMind称，DNC还可以帮你规划从沼泽门到皮卡迪利广场的最佳路线。

          同样，它还可以理解和回答某个大家族中的成员之间的关系这样的复杂问题，比如“张三的大舅是谁？”。

          DNC建立在神经网络的概念之上，神经网络可以模拟人类思想活动的方式。这种AI技术很适合与机器习得配套使用。

          DeepMind的AlphaGo AI能够打败围棋冠军也跟这些神经网络有很大关系。但是AlphaGo必须进行训练才行，开发人员向AlphaGo提供了历史对弈中的大约3000万记录。让人工智能技术具备通过记忆学习的能力，就可以让它独自完成更复杂的任务。

          DeepMind希望DNC可以推动计算行业实现更多突破。DeepMind已将其研究结果发表在科学刊物《自然》（Nature）上。（编译/林靖东）

          精彩视频推荐

          自动播放开关 自动播放

          全民微信时代，用语音功能发60秒是种怎样的体验？

          转播到腾讯微博
          正在加载...

            【美国The Next Web作品的中文相关权益归腾讯公司独家所有。未经授权，不得转载、摘编等。】

            [责任编辑：alonliu]
            我要评论(0)
            您认为这篇文章与"新一网（08008.HK）"相关度高吗？

            更多评论

            ================================================ FILE: test/test-pages/quanta-1/expected-metadata.json ================================================ { "title": "The Hidden Heroines of Chaos", "byline": "By Joshua Sokol", "dir": null, "lang": "en", "excerpt": "Two women programmers played a pivotal role in the birth of chaos theory. Their previously untold story illustrates the changing status of computation in", "siteName": "Quanta Magazine", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/quanta-1/expected.html ================================================

            A little over half a century ago, chaos started spilling out of a famous experiment. It came not from a petri dish, a beaker or an astronomical observatory, but from the vacuum tubes and diodes of a Royal McBee LGP-30. This “desk” computer — it was the size of a desk — weighed some 800 pounds and sounded like a passing propeller plane. It was so loud that it even got its own office on the fifth floor in Building 24, a drab structure near the center of the Massachusetts Institute of Technology. Instructions for the computer came from down the hall, from the office of a meteorologist named Edward Norton Lorenz.

            The story of chaos is usually told like this: Using the LGP-30, Lorenz made paradigm-wrecking discoveries. In 1961, having programmed a set of equations into the computer that would simulate future weather, he found that tiny differences in starting values could lead to drastically different outcomes. This sensitivity to initial conditions, later popularized as the butterfly effect, made predicting the far future a fool’s errand. But Lorenz also found that these unpredictable outcomes weren’t quite random, either. When visualized in a certain way, they seemed to prowl around a shape called a strange attractor.

            About a decade later, chaos theory started to catch on in scientific circles. Scientists soon encountered other unpredictable natural systems that looked random even though they weren’t: the rings of Saturn, blooms of marine algae, Earth’s magnetic field, the number of salmon in a fishery. Then chaos went mainstream with the publication of James Gleick’s Chaos: Making a New Science in 1987. Before long, Jeff Goldblum, playing the chaos theorist Ian Malcolm, was pausing, stammering and charming his way through lines about the unpredictability of nature in Jurassic Park.

            All told, it’s a neat narrative. Lorenz, “the father of chaos,” started a scientific revolution on the LGP-30. It is quite literally a textbook case for how the numerical experiments that modern science has come to rely on — in fields ranging from climate science to ecology to astrophysics — can uncover hidden truths about nature.

            But in fact, Lorenz was not the one running the machine. There’s another story, one that has gone untold for half a century. A year and a half ago, an MIT scientist happened across a name he had never heard before and started to investigate. The trail he ended up following took him into the MIT archives, through the stacks of the Library of Congress, and across three states and five decades to find information about the women who, today, would have been listed as co-authors on that seminal paper. And that material, shared with Quanta, provides a fuller, fairer account of the birth of chaos.

            The Birth of Chaos

            In the fall of 2017, the geophysicist Daniel Rothman, co-director of MIT’s Lorenz Center, was preparing for an upcoming symposium. The meeting would honor Lorenz, who died in 2008, so Rothman revisited Lorenz’s epochal paper, a masterwork on chaos titled “Deterministic Nonperiodic Flow.” Published in 1963, it has since attracted thousands of citations, and Rothman, having taught this foundational material to class after class, knew it like an old friend. But this time he saw something he hadn’t noticed before. In the paper’s acknowledgments, Lorenz had written, “Special thanks are due to Miss Ellen Fetter for handling the many numerical computations.”

            “Jesus … who is Ellen Fetter?” Rothman recalls thinking at the time. “It’s one of the most important papers in computational physics and, more broadly, in computational science,” he said. And yet he couldn’t find anything about this woman. “Of all the volumes that have been written about Lorenz, the great discovery — nothing.”

            With further online searches, however, Rothman found a wedding announcement from 1963. Ellen Fetter had married John Gille, a physicist, and changed her name. A colleague of Rothman’s then remembered that a graduate student named Sarah Gille had studied at MIT in the 1990s in the very same department as Lorenz and Rothman. Rothman reached out to her, and it turned out that Sarah Gille, now a physical oceanographer at the University of California, San Diego, was Ellen and John’s daughter. Through this connection, Rothman was able to get Ellen Gille, née Fetter, on the phone. And that’s when he learned another name, the name of the woman who had preceded Fetter in the job of programming Lorenz’s first meetings with chaos: Margaret Hamilton.

            When Margaret Hamilton arrived at MIT in the summer of 1959, with a freshly minted math degree from Earlham College, Lorenz had only recently bought and taught himself to use the LGP-30. Hamilton had no prior training in programming either. Then again, neither did anyone else at the time. “He loved that computer,” Hamilton said. “And he made me feel the same way about it.”

            For Hamilton, these were formative years. She recalls being out at a party at three or four a.m., realizing that the LGP-30 wasn’t set to produce results by the next morning, and rushing over with a few friends to start it up. Another time, frustrated by all the things that had to be done to make another run after fixing an error, she devised a way to bypass the computer’s clunky debugging process. To Lorenz’s delight, Hamilton would take the paper tape that fed the machine, roll it out the length of the hallway, and edit the binary code with a sharp pencil. “I’d poke holes for ones, and I’d cover up with Scotch tape the others,” she said. “He just got a kick out of it.”

            There were desks in the computer room, but because of the noise, Lorenz, his secretary, his programmer and his graduate students all shared the other office. The plan was to use the desk computer, then a total novelty, to test competing strategies of weather prediction in a way you couldn’t do with pencil and paper.

            First, though, Lorenz’s team had to do the equivalent of catching the Earth’s atmosphere in a jar. Lorenz idealized the atmosphere in 12 equations that described the motion of gas in a rotating, stratified fluid. Then the team coded them in.

            Sometimes the “weather” inside this simulation would simply repeat like clockwork. But Lorenz found a more interesting and more realistic set of solutions that generated weather that wasn’t periodic. The team set up the computer to slowly print out a graph of how one or two variables — say, the latitude of the strongest westerly winds — changed over time. They would gather around to watch this imaginary weather, even placing little bets on what the program would do next.

            And then one day it did something really strange. This time they had set up the printer not to make a graph, but simply to print out time stamps and the values of a few variables at each time. As Lorenz later recalled, they had re-run a previous weather simulation with what they thought were the same starting values, reading off the earlier numbers from the previous printout. But those weren’t actually the same numbers. The computer was keeping track of numbers to six decimal places, but the printer, to save space on the page, had rounded them to only the first three decimal places.

            After the second run started, Lorenz went to get coffee. The new numbers that emerged from the LGP-30 while he was gone looked at first like the ones from the previous run. This new run had started in a very similar place, after all. But the errors grew exponentially. After about two months of imaginary weather, the two runs looked nothing alike. This system was still deterministic, with no random chance intruding between one moment and the next. Even so, its hair-trigger sensitivity to initial conditions made it unpredictable.

            This meant that in chaotic systems the smallest fluctuations get amplified. Weather predictions fail once they reach some point in the future because we can never measure the initial state of the atmosphere precisely enough. Or, as Lorenz would later present the idea, even a seagull flapping its wings might eventually make a big difference to the weather. (In 1972, the seagull was deposed when a conference organizer, unable to check back about what Lorenz wanted to call an upcoming talk, wrote his own title that switched the metaphor to a butterfly.)

            Many accounts, including the one in Gleick’s book, date the discovery of this butterfly effect to 1961, with the paper following in 1963. But in November 1960, Lorenz described it during the Q&A session following a talk he gave at a conference on numerical weather prediction in Tokyo. After his talk, a question came from a member of the audience: “Did you change the initial condition just slightly and see how much different results were?”

            “As a matter of fact, we tried out that once with the same equation to see what could happen,” Lorenz said. He then started to explain the unexpected result, which he wouldn’t publish for three more years. “He just gives it all away,” Rothman said now. But no one at the time registered it enough to scoop him.

            In the summer of 1961, Hamilton moved on to another project, but not before training her replacement. Two years after Hamilton first stepped on campus, Ellen Fetter showed up at MIT in much the same fashion: a recent graduate of Mount Holyoke with a degree in math, seeking any sort of math-related job in the Boston area, eager and able to learn. She interviewed with a woman who ran the LGP-30 in the nuclear engineering department, who recommended her to Hamilton, who hired her.

            Once Fetter arrived in Building 24, Lorenz gave her a manual and a set of programming problems to practice, and before long she was up to speed. “He carried a lot in his head,” she said. “He would come in with maybe one yellow sheet of paper, a legal piece of paper in his pocket, pull it out, and say, ‘Let’s try this.’”

            The project had progressed meanwhile. The 12 equations produced fickle weather, but even so, that weather seemed to prefer a narrow set of possibilities among all possible states, forming a mysterious cluster which Lorenz wanted to visualize. Finding that difficult, he narrowed his focus even further. From a colleague named Barry Saltzman, he borrowed just three equations that would describe an even simpler nonperiodic system, a beaker of water heated from below and cooled from above.

            Here, again, the LGP-30 chugged its way into chaos. Lorenz identified three properties of the system corresponding roughly to how fast convection was happening in the idealized beaker, how the temperature varied from side to side, and how the temperature varied from top to bottom. The computer tracked these properties moment by moment.

            The properties could also be represented as a point in space. Lorenz and Fetter plotted the motion of this point. They found that over time, the point would trace out a butterfly-shaped fractal structure now called the Lorenz attractor. The trajectory of the point — of the system — would never retrace its own path. And as before, two systems setting out from two minutely different starting points would soon be on totally different tracks. But just as profoundly, wherever you started the system, it would still head over to the attractor and start doing chaotic laps around it.

            The attractor and the system’s sensitivity to initial conditions would eventually be recognized as foundations of chaos theory. Both were published in the landmark 1963 paper. But for a while only meteorologists noticed the result. Meanwhile, Fetter married John Gille and moved with him when he went to Florida State University and then to Colorado. They stayed in touch with Lorenz and saw him at social events. But she didn’t realize how famous he had become.

            Still, the notion of small differences leading to drastically different outcomes stayed in the back of her mind. She remembered the seagull, flapping its wings. “I always had this image that stepping off the curb one way or the other could change the course of any field,” she said.

            Flight Checks

            After leaving Lorenz’s group, Hamilton embarked on a different path, achieving a level of fame that rivals or even exceeds that of her first coding mentor. At MIT’s Instrumentation Laboratory, starting in 1965, she headed the onboard flight software team for the Apollo project.

            Her code held up when the stakes were life and death — even when a mis-flipped switch triggered alarms that interrupted the astronaut’s displays right as Apollo 11 approached the surface of the moon. Mission Control had to make a quick choice: land or abort. But trusting the software’s ability to recognize errors, prioritize important tasks, and recover, the astronauts kept going.

            Hamilton, who popularized the term “software engineering,” later led the team that wrote the software for Skylab, the first U.S. space station. She founded her own company in Cambridge in 1976, and in recent years her legacy has been celebrated again and again. She won NASA’s Exceptional Space Act Award in 2003 and received the Presidential Medal of Freedom in 2016. In 2017 she garnered arguably the greatest honor of all: a Margaret Hamilton Lego minifigure.

            Fetter, for her part, continued to program at Florida State after leaving Lorenz’s group at MIT. After a few years, she left her job to raise her children. In the 1970s, she took computer science classes at the University of Colorado, toying with the idea of returning to programming, but she eventually took a tax preparation job instead. By the 1980s, the demographics of programming had shifted. “After I sort of got put off by a couple of job interviews, I said forget it,” she said. “They went with young, techy guys.”

            Chaos only reentered her life through her daughter, Sarah. As an undergraduate at Yale in the 1980s, Sarah Gille sat in on a class about scientific programming. The case they studied? Lorenz’s discoveries on the LGP-30. Later, Sarah studied physical oceanography as a graduate student at MIT, joining the same overarching department as both Lorenz and Rothman, who had arrived a few years earlier. “One of my office mates in the general exam, the qualifying exam for doing research at MIT, was asked: How would you explain chaos theory to your mother?” she said. “I was like, whew, glad I didn’t get that question.”

            The Changing Value of Computation

            Today, chaos theory is part of the scientific repertoire. In a study published just last month, researchers concluded that no amount of improvement in data gathering or in the science of weather forecasting will allow meteorologists to produce useful forecasts that stretch more than 15 days out. (Lorenz had suggested a similar two-week cap to weather forecasts in the mid-1960s.)

            But the many retellings of chaos’s birth say little to nothing about how Hamilton and Ellen Gille wrote the specific programs that revealed the signatures of chaos. “This is an all-too-common story in the histories of science and technology,” wrote Jennifer Light, the department head for MIT’s Science, Technology and Society program, in an email to Quanta. To an extent, we can chalk up that omission to the tendency of storytellers to focus on solitary geniuses. But it also stems from tensions that remain unresolved today.

            First, coders in general have seen their contributions to science minimized from the beginning. “It was seen as rote,” said Mar Hicks, a historian at the Illinois Institute of Technology. “The fact that it was associated with machines actually gave it less status, rather than more.” But beyond that, and contributing to it, many programmers in this era were women.

            In addition to Hamilton and the woman who coded in MIT’s nuclear engineering department, Ellen Gille recalls a woman on an LGP-30 doing meteorology next door to Lorenz’s group. Another woman followed Gille in the job of programming for Lorenz. An analysis of official U.S. labor statistics shows that in 1960, women held 27 percent of computing and math-related jobs.

            The percentage has been stuck there for a half-century. In the mid-1980s, the fraction of women pursuing bachelor’s degrees in programming even started to decline. Experts have argued over why. One idea holds that early personal computers were marketed preferentially to boys and men. Then when kids went to college, introductory classes assumed a detailed knowledge of computers going in, which alienated young women who didn’t grow up with a machine at home. Today, women programmers describe a self-perpetuating cycle where white and Asian male managers hire people who look like all the other programmers they know. Outright harassment also remains a problem.

            Hamilton and Gille, however, still speak of Lorenz’s humility and mentorship in glowing terms. Before later chroniclers left them out, Lorenz thanked them in the literature in the same way he thanked Saltzman, who provided the equations Lorenz used to find his attractor. This was common at the time. Gille recalls that in all her scientific programming work, only once did someone include her as a co-author after she contributed computational work to a paper; she said she was “stunned” because of how unusual that was.

            Since then, the standard for giving credit has shifted. “If you went up and down the floors of this building and told the story to my colleagues, every one of them would say that if this were going on today … they’d be a co-author!” Rothman said. “Automatically, they’d be a co-author.”

            Computation in science has become even more indispensable, of course. For recent breakthroughs like the first image of a black hole, the hard part was not figuring out which equations described the system, but how to leverage computers to understand the data.

            Today, many programmers leave science not because their role isn’t appreciated, but because coding is better compensated in industry, said Alyssa Goodman, an astronomer at Harvard University and an expert in computing and data science. “In the 1960s, there was no such thing as a data scientist, there was no such thing as Netflix or Google or whoever, that was going to suck in these people and really, really value them,” she said.

            Still, for coder-scientists in academic systems that measure success by paper citations, things haven’t changed all that much. “If you are a software developer who may never write a paper, you may be essential,” Goodman said. “But you’re not going to be counted that way.”

            This article was reprinted on Wired.com.

            ================================================ FILE: test/test-pages/quanta-1/source.html ================================================ Quanta Magazine
            chaos theory

            The Hidden Heroines of Chaos

            By Joshua Sokol

            May 20, 2019

            Two women programmers played a pivotal role in the birth of chaos theory. Their previously untold story illustrates the changing status of computation in science.
            Animated line drawing of Margaret Hamilton, Ellen Fetter, and a Lorenz attractor

            Ellen Fetter and Margaret Hamilton were responsible for programming the enormous 1960s-era computer that would uncover strange attractors and other hallmarks of chaos theory.

            Olena Shmahalo/Quanta Magazine

            A little over half a century ago, chaos started spilling out of a famous experiment. It came not from a petri dish, a beaker or an astronomical observatory, but from the vacuum tubes and diodes of a Royal McBee LGP-30. This “desk” computer — it was the size of a desk — weighed some 800 pounds and sounded like a passing propeller plane. It was so loud that it even got its own office on the fifth floor in Building 24, a drab structure near the center of the Massachusetts Institute of Technology. Instructions for the computer came from down the hall, from the office of a meteorologist named Edward Norton Lorenz.

            The story of chaos is usually told like this: Using the LGP-30, Lorenz made paradigm-wrecking discoveries. In 1961, having programmed a set of equations into the computer that would simulate future weather, he found that tiny differences in starting values could lead to drastically different outcomes. This sensitivity to initial conditions, later popularized as the butterfly effect, made predicting the far future a fool’s errand. But Lorenz also found that these unpredictable outcomes weren’t quite random, either. When visualized in a certain way, they seemed to prowl around a shape called a strange attractor.

            About a decade later, chaos theory started to catch on in scientific circles. Scientists soon encountered other unpredictable natural systems that looked random even though they weren’t: the rings of Saturn, blooms of marine algae, Earth’s magnetic field, the number of salmon in a fishery. Then chaos went mainstream with the publication of James Gleick’s Chaos: Making a New Science in 1987. Before long, Jeff Goldblum, playing the chaos theorist Ian Malcolm, was pausing, stammering and charming his way through lines about the unpredictability of nature in Jurassic Park.

            All told, it’s a neat narrative. Lorenz, “the father of chaos,” started a scientific revolution on the LGP-30. It is quite literally a textbook case for how the numerical experiments that modern science has come to rely on — in fields ranging from climate science to ecology to astrophysics — can uncover hidden truths about nature.

            But in fact, Lorenz was not the one running the machine. There’s another story, one that has gone untold for half a century. A year and a half ago, an MIT scientist happened across a name he had never heard before and started to investigate. The trail he ended up following took him into the MIT archives, through the stacks of the Library of Congress, and across three states and five decades to find information about the women who, today, would have been listed as co-authors on that seminal paper. And that material, shared with Quanta, provides a fuller, fairer account of the birth of chaos.

            The Birth of Chaos

            In the fall of 2017, the geophysicist Daniel Rothman, co-director of MIT’s Lorenz Center, was preparing for an upcoming symposium. The meeting would honor Lorenz, who died in 2008, so Rothman revisited Lorenz’s epochal paper, a masterwork on chaos titled “Deterministic Nonperiodic Flow.” Published in 1963, it has since attracted thousands of citations, and Rothman, having taught this foundational material to class after class, knew it like an old friend. But this time he saw something he hadn’t noticed before. In the paper’s acknowledgments, Lorenz had written, “Special thanks are due to Miss Ellen Fetter for handling the many numerical computations.”

            “Jesus … who is Ellen Fetter?” Rothman recalls thinking at the time. “It’s one of the most important papers in computational physics and, more broadly, in computational science,” he said. And yet he couldn’t find anything about this woman. “Of all the volumes that have been written about Lorenz, the great discovery — nothing.”

            With further online searches, however, Rothman found a wedding announcement from 1963. Ellen Fetter had married John Gille, a physicist, and changed her name. A colleague of Rothman’s then remembered that a graduate student named Sarah Gille had studied at MIT in the 1990s in the very same department as Lorenz and Rothman. Rothman reached out to her, and it turned out that Sarah Gille, now a physical oceanographer at the University of California, San Diego, was Ellen and John’s daughter. Through this connection, Rothman was able to get Ellen Gille, née Fetter, on the phone. And that’s when he learned another name, the name of the woman who had preceded Fetter in the job of programming Lorenz’s first meetings with chaos: Margaret Hamilton.

            When Margaret Hamilton arrived at MIT in the summer of 1959, with a freshly minted math degree from Earlham College, Lorenz had only recently bought and taught himself to use the LGP-30. Hamilton had no prior training in programming either. Then again, neither did anyone else at the time. “He loved that computer,” Hamilton said. “And he made me feel the same way about it.”

            For Hamilton, these were formative years. She recalls being out at a party at three or four a.m., realizing that the LGP-30 wasn’t set to produce results by the next morning, and rushing over with a few friends to start it up. Another time, frustrated by all the things that had to be done to make another run after fixing an error, she devised a way to bypass the computer’s clunky debugging process. To Lorenz’s delight, Hamilton would take the paper tape that fed the machine, roll it out the length of the hallway, and edit the binary code with a sharp pencil. “I’d poke holes for ones, and I’d cover up with Scotch tape the others,” she said. “He just got a kick out of it.”

            Acknowledgements of Ellen Fetter and Margaret Hamilton in Edward Lorenz' 1963 and 1962 papers

            Edward Lorenz acknowledged the contributions of Fetter and Hamilton at the end of his papers.

            There were desks in the computer room, but because of the noise, Lorenz, his secretary, his programmer and his graduate students all shared the other office. The plan was to use the desk computer, then a total novelty, to test competing strategies of weather prediction in a way you couldn’t do with pencil and paper.

            First, though, Lorenz’s team had to do the equivalent of catching the Earth’s atmosphere in a jar. Lorenz idealized the atmosphere in 12 equations that described the motion of gas in a rotating, stratified fluid. Then the team coded them in.

            Sometimes the “weather” inside this simulation would simply repeat like clockwork. But Lorenz found a more interesting and more realistic set of solutions that generated weather that wasn’t periodic. The team set up the computer to slowly print out a graph of how one or two variables — say, the latitude of the strongest westerly winds — changed over time. They would gather around to watch this imaginary weather, even placing little bets on what the program would do next.

            And then one day it did something really strange. This time they had set up the printer not to make a graph, but simply to print out time stamps and the values of a few variables at each time. As Lorenz later recalled, they had re-run a previous weather simulation with what they thought were the same starting values, reading off the earlier numbers from the previous printout. But those weren’t actually the same numbers. The computer was keeping track of numbers to six decimal places, but the printer, to save space on the page, had rounded them to only the first three decimal places.

            After the second run started, Lorenz went to get coffee. The new numbers that emerged from the LGP-30 while he was gone looked at first like the ones from the previous run. This new run had started in a very similar place, after all. But the errors grew exponentially. After about two months of imaginary weather, the two runs looked nothing alike. This system was still deterministic, with no random chance intruding between one moment and the next. Even so, its hair-trigger sensitivity to initial conditions made it unpredictable.

            This meant that in chaotic systems the smallest fluctuations get amplified. Weather predictions fail once they reach some point in the future because we can never measure the initial state of the atmosphere precisely enough. Or, as Lorenz would later present the idea, even a seagull flapping its wings might eventually make a big difference to the weather. (In 1972, the seagull was deposed when a conference organizer, unable to check back about what Lorenz wanted to call an upcoming talk, wrote his own title that switched the metaphor to a butterfly.)

            "A Brief History of Chaos" Infographic

            5W Infographics for Quanta Magazine; sources: Wikimedia Commons,
            Mathematical Association of America, MIT Museum

            Many accounts, including the one in Gleick’s book, date the discovery of this butterfly effect to 1961, with the paper following in 1963. But in November 1960, Lorenz described it during the Q&A session following a talk he gave at a conference on numerical weather prediction in Tokyo. After his talk, a question came from a member of the audience: “Did you change the initial condition just slightly and see how much different results were?”

            “As a matter of fact, we tried out that once with the same equation to see what could happen,” Lorenz said. He then started to explain the unexpected result, which he wouldn’t publish for three more years. “He just gives it all away,” Rothman said now. But no one at the time registered it enough to scoop him.

            In the summer of 1961, Hamilton moved on to another project, but not before training her replacement. Two years after Hamilton first stepped on campus, Ellen Fetter showed up at MIT in much the same fashion: a recent graduate of Mount Holyoke with a degree in math, seeking any sort of math-related job in the Boston area, eager and able to learn. She interviewed with a woman who ran the LGP-30 in the nuclear engineering department, who recommended her to Hamilton, who hired her.

            Once Fetter arrived in Building 24, Lorenz gave her a manual and a set of programming problems to practice, and before long she was up to speed. “He carried a lot in his head,” she said. “He would come in with maybe one yellow sheet of paper, a legal piece of paper in his pocket, pull it out, and say, ‘Let’s try this.’”

            The project had progressed meanwhile. The 12 equations produced fickle weather, but even so, that weather seemed to prefer a narrow set of possibilities among all possible states, forming a mysterious cluster which Lorenz wanted to visualize. Finding that difficult, he narrowed his focus even further. From a colleague named Barry Saltzman, he borrowed just three equations that would describe an even simpler nonperiodic system, a beaker of water heated from below and cooled from above.

            Here, again, the LGP-30 chugged its way into chaos. Lorenz identified three properties of the system corresponding roughly to how fast convection was happening in the idealized beaker, how the temperature varied from side to side, and how the temperature varied from top to bottom. The computer tracked these properties moment by moment.

            The properties could also be represented as a point in space. Lorenz and Fetter plotted the motion of this point. They found that over time, the point would trace out a butterfly-shaped fractal structure now called the Lorenz attractor. The trajectory of the point — of the system — would never retrace its own path. And as before, two systems setting out from two minutely different starting points would soon be on totally different tracks. But just as profoundly, wherever you started the system, it would still head over to the attractor and start doing chaotic laps around it.

            The attractor and the system’s sensitivity to initial conditions would eventually be recognized as foundations of chaos theory. Both were published in the landmark 1963 paper. But for a while only meteorologists noticed the result. Meanwhile, Fetter married John Gille and moved with him when he went to Florida State University and then to Colorado. They stayed in touch with Lorenz and saw him at social events. But she didn’t realize how famous he had become.

            Still, the notion of small differences leading to drastically different outcomes stayed in the back of her mind. She remembered the seagull, flapping its wings. “I always had this image that stepping off the curb one way or the other could change the course of any field,” she said.

            Flight Checks

            After leaving Lorenz’s group, Hamilton embarked on a different path, achieving a level of fame that rivals or even exceeds that of her first coding mentor. At MIT’s Instrumentation Laboratory, starting in 1965, she headed the onboard flight software team for the Apollo project.

            Her code held up when the stakes were life and death — even when a mis-flipped switch triggered alarms that interrupted the astronaut’s displays right as Apollo 11 approached the surface of the moon. Mission Control had to make a quick choice: land or abort. But trusting the software’s ability to recognize errors, prioritize important tasks, and recover, the astronauts kept going.

            Hamilton, who popularized the term “software engineering,” later led the team that wrote the software for Skylab, the first U.S. space station. She founded her own company in Cambridge in 1976, and in recent years her legacy has been celebrated again and again. She won NASA’s Exceptional Space Act Award in 2003 and received the Presidential Medal of Freedom in 2016. In 2017 she garnered arguably the greatest honor of all: a Margaret Hamilton Lego minifigure.

            Fetter, for her part, continued to program at Florida State after leaving Lorenz’s group at MIT. After a few years, she left her job to raise her children. In the 1970s, she took computer science classes at the University of Colorado, toying with the idea of returning to programming, but she eventually took a tax preparation job instead. By the 1980s, the demographics of programming had shifted. “After I sort of got put off by a couple of job interviews, I said forget it,” she said. “They went with young, techy guys.”

            Chaos only reentered her life through her daughter, Sarah. As an undergraduate at Yale in the 1980s, Sarah Gille sat in on a class about scientific programming. The case they studied? Lorenz’s discoveries on the LGP-30. Later, Sarah studied physical oceanography as a graduate student at MIT, joining the same overarching department as both Lorenz and Rothman, who had arrived a few years earlier. “One of my office mates in the general exam, the qualifying exam for doing research at MIT, was asked: How would you explain chaos theory to your mother?” she said. “I was like, whew, glad I didn’t get that question.”

            The Changing Value of Computation

            Today, chaos theory is part of the scientific repertoire. In a study published just last month, researchers concluded that no amount of improvement in data gathering or in the science of weather forecasting will allow meteorologists to produce useful forecasts that stretch more than 15 days out. (Lorenz had suggested a similar two-week cap to weather forecasts in the mid-1960s.)

            But the many retellings of chaos’s birth say little to nothing about how Hamilton and Ellen Gille wrote the specific programs that revealed the signatures of chaos. “This is an all-too-common story in the histories of science and technology,” wrote Jennifer Light, the department head for MIT’s Science, Technology and Society program, in an email to Quanta. To an extent, we can chalk up that omission to the tendency of storytellers to focus on solitary geniuses. But it also stems from tensions that remain unresolved today.

            First, coders in general have seen their contributions to science minimized from the beginning. “It was seen as rote,” said Mar Hicks, a historian at the Illinois Institute of Technology. “The fact that it was associated with machines actually gave it less status, rather than more.” But beyond that, and contributing to it, many programmers in this era were women.

            In addition to Hamilton and the woman who coded in MIT’s nuclear engineering department, Ellen Gille recalls a woman on an LGP-30 doing meteorology next door to Lorenz’s group. Another woman followed Gille in the job of programming for Lorenz. An analysis of official U.S. labor statistics shows that in 1960, women held 27 percent of computing and math-related jobs.

            The percentage has been stuck there for a half-century. In the mid-1980s, the fraction of women pursuing bachelor’s degrees in programming even started to decline. Experts have argued over why. One idea holds that early personal computers were marketed preferentially to boys and men. Then when kids went to college, introductory classes assumed a detailed knowledge of computers going in, which alienated young women who didn’t grow up with a machine at home. Today, women programmers describe a self-perpetuating cycle where white and Asian male managers hire people who look like all the other programmers they know. Outright harassment also remains a problem.

            Hamilton and Gille, however, still speak of Lorenz’s humility and mentorship in glowing terms. Before later chroniclers left them out, Lorenz thanked them in the literature in the same way he thanked Saltzman, who provided the equations Lorenz used to find his attractor. This was common at the time. Gille recalls that in all her scientific programming work, only once did someone include her as a co-author after she contributed computational work to a paper; she said she was “stunned” because of how unusual that was.

            Since then, the standard for giving credit has shifted. “If you went up and down the floors of this building and told the story to my colleagues, every one of them would say that if this were going on today … they’d be a co-author!” Rothman said. “Automatically, they’d be a co-author.”

            Computation in science has become even more indispensable, of course. For recent breakthroughs like the first image of a black hole, the hard part was not figuring out which equations described the system, but how to leverage computers to understand the data.

            Today, many programmers leave science not because their role isn’t appreciated, but because coding is better compensated in industry, said Alyssa Goodman, an astronomer at Harvard University and an expert in computing and data science. “In the 1960s, there was no such thing as a data scientist, there was no such thing as Netflix or Google or whoever, that was going to suck in these people and really, really value them,” she said.

            Still, for coder-scientists in academic systems that measure success by paper citations, things haven’t changed all that much. “If you are a software developer who may never write a paper, you may be essential,” Goodman said. “But you’re not going to be counted that way.”

            This article was reprinted on Wired.com.

            Comment on this article

            Quanta Magazine moderates comments to facilitate an informed, substantive, civil conversation. Abusive, profane, self-promotional, misleading, incoherent or off-topic comments will be rejected. Moderators are staffed during regular business hours (New York time) and can only accept comments written in English. 

            ================================================ FILE: test/test-pages/remove-aria-hidden/expected-metadata.json ================================================ { "title": "Remove aria-hidden elements test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": false } ================================================ FILE: test/test-pages/remove-aria-hidden/expected.html ================================================

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            ================================================ FILE: test/test-pages/remove-aria-hidden/source.html ================================================  Remove aria-hidden elements test

            Lorem

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            ================================================ FILE: test/test-pages/remove-extra-brs/expected-metadata.json ================================================ { "title": "Remove trailing brs test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/remove-extra-brs/expected.html ================================================

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/remove-extra-brs/source.html ================================================ Remove trailing brs test

            Lorem


            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.



            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.


            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/remove-extra-paragraphs/expected-metadata.json ================================================ { "title": "Replace font tags test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/remove-extra-paragraphs/expected.html ================================================

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/remove-extra-paragraphs/source.html ================================================ Replace font tags test

            Lorem

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/remove-script-tags/expected-metadata.json ================================================ { "title": "Remove script tags test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/remove-script-tags/expected.html ================================================

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/remove-script-tags/source.html ================================================ Remove script tags test

            Lorem

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

            Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

            Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/reordering-paragraphs/expected-metadata.json ================================================ { "title": "", "byline": null, "dir": null, "excerpt": "Regarding item# 11111, under sufficiently extreme conditions, quarks may\n become deconfined and exist as free particles. In the course of asymptotic\n freedom, the strong interaction becomes weaker at higher temperatures.\n Eventually, color confinement would be lost and an extremely hot plasma\n of freely moving quarks and gluons would be formed. This theoretical phase\n of matter is called quark-gluon plasma.[81] The exact conditions needed\n to give rise to this state are unknown and have been the subject of a great\n deal of speculation and experimentation.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/reordering-paragraphs/expected.html ================================================

            Regarding item# 11111, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

            Regarding item# 22222, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

            Regarding item# 33333, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.


            ================================================ FILE: test/test-pages/reordering-paragraphs/source.html ================================================

            Regarding item# 11111, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

            Regarding item# 22222, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

            Regarding item# 33333, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.


            ================================================ FILE: test/test-pages/replace-brs/expected-metadata.json ================================================ { "title": "Replace brs test", "byline": null, "dir": null, "excerpt": "Lorem ipsumdolor sit", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/replace-brs/expected.html ================================================

            Lorem ipsum
            dolor sit

            amet, consectetur adipisicing elit, sed do eiusmod
            tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
            quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo

            consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
            cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
            proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor

            incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
            quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
            consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse

            cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
            proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/replace-brs/source.html ================================================ Replace brs test

            Lorem

            Lorem ipsum
            dolor sit


            amet, consectetur adipisicing elit, sed do eiusmod
            tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
            quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo

            consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
            cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
            proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor

            incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
            quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
            consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse

            cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
            proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
            ================================================ FILE: test/test-pages/replace-font-tags/expected-metadata.json ================================================ { "title": "Replace font tags test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/replace-font-tags/expected.html ================================================

            Lorem

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/replace-font-tags/source.html ================================================ Replace font tags test

            Lorem

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Foo

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
            ================================================ FILE: test/test-pages/royal-road/expected-metadata.json ================================================ { "title": "ONE HUNDRED TWO: What kind of wordchain? - Super Supportive", "byline": "Follow Author", "dir": null, "lang": "en", "excerpt": "102 “Were you expecting the competition for the showers to be the highest drama part of gym class?” Alden asked Haoyu as the two of them headed (...)", "siteName": "Royal Road", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/royal-road/expected.html ================================================

            102

            “Were you expecting the competition for the showers to be the highest drama part of gym class?” Alden asked Haoyu as the two of them headed down a broad avenue toward North of North gym. It was a ten minute walk from the MPE building, right on the edge of campus, and apparently, they had both had the same idea when they discovered they were going to have to wait for their turns anyway.

            Haoyu was gulping from a water bottle. “I’m glad my group was getting along well. I think the one with the speedsters in it had some kind of issue. They seemed tense.”

            Finlay had looked pretty mad. It was the first time Alden had seen the Scottish boy in anything but a cheerful mood.

            “I didn’t have the attention span to watch them and run from Snake’s tennis ball pitches. It somehow adds insult to injury that he makes you chase the balls down after they hit you.”

            When they reached the airy glass building that Alden thought of as the main gym, the same girl in lime green yoga pants who’d been protecting the entrance from riffraff and gawkers on his first visit greeted them both warmly. She was giving out cups of some of the liquefied salads that the spa upstairs bottled and sold as drinks.

            They both took one.

            As they headed toward the showers, Haoyu sipped the black pepper beet juice he’d chosen and wrinkled his nose. “I don’t think I’m old enough to drink this.”

            Alden raised an eyebrow at him.

            “I’m trying!” Haoyu said. “Being friends with Lexi’s hard. He got really serious about nutrition in the past year or two. I saw him cut a single piece of chocolate in half once. And there I was stuffing six in my mouth at a time. I want to eat healthy, but I like candy so much. And anything fried.”

            “Is that why you bought the slow cooker?”

            Haoyu nodded. “That, and because my mom kept telling me I shouldn’t buy it because I wouldn’t ever use it. I’m going to use it every day. And send her pictures. Starting tonight. The bouncing oatmeal was just a temporary setback. It didn’t even taste that bad.”

            He looked curiously at a man casting spells on a section of the climbing wall as they passed it. “I’m so glad you’ve got a membership here, too. I really wanted one, and my parents wanted me to have it. But I almost told them not to buy it for me because it’s so…um…”

            “Richy?”

            “Yes. Mom and Dad both come here, but they have to. They need the facilities in the heavy gym. I could probably have used the school equipment until I was halfway through uni. But everything here is so much nicer, and I used to play on the rock wall in the onsite daycare and just imagine how great it was going to be when I was finally fifteen and I could use the real one. They can set it to have high winds, floods, and earthquakes! I went for it. And you’re here, too! We’ll both look richy together, and it won’t be as awkward when we leave school to use the better stuff.”

            “Happy to spoil myself with you.”

            They walked past an attendant into the room with the big rainfall showers and the ridiculously fluffy scented towels.

            Haoyu beamed at him. “Soooo, since you’re here with me, and you’re not going to freak out about it…I’ll confess that I actually already have potion therapy sauna slots booked for a few days a week until the end of term. I was always planning to come here instead of using the student showers.”

            Alden blinked at him. The potions saunas had been a part of his facilities tour. For a hundred argold—or more depending on which sauna you chose—you could sweat and inhale magical substances that did rather out-there sounding things.

            “Are you having your spirit waxed or…?”

            “No!” Haoyu laughed. “Is that even real? I’m using the recovery sauna. Potions aren’t as good as healers, obviously, but if I sit in there for ninety minutes or so, I don’t really need to rest on my rest days, and I don’t get too sore. I’m just going to do my homework in there instead of sitting through study halls. My mom did sauna homework her whole last year of uni. It was her idea, and since I wasn’t saying no to the gym membership I wasn’t going to say no to something like this.”

            He’s basically turning the two hour study hall into recovery time. Alden wondered if Mrs. Zhang-Demir had been attempting to balance out Haoyu’s lifestyle when she tried to set a dorm decorating budget.

            “That’s very efficient,” said Alden.

            “I’m telling you so you can book a slot tonight, if you want to do homework together. And so you’re not like, ‘Where did Haoyu go? Is he trying to get away from me?’ when I jump straight out of the shower and run off to the sauna instead of heading back to the dorms with you.”

            “Gotcha.”

            “Do you want to come?”

            “…isn’t it a lot of famous people sitting around in towels together, talking about superhero stuff?”

            That sounded like a high pressure environment to do homework in unless your parents were famous superheroes and you were used to them.

            “I’ve only used it a few times so far. It’s not that many people, and they’re all zoned out watching videos or working on their interfaces. They don’t talk and interrupt my studying, so it’s good.”

            “Let me think about it.”

            ******

            Alden had a couple of minutes to debate it while they both rinsed off.

            Pro: doing homework with Haoyu was in line with his mission to be a great roommate.

            Con: shirtlessness.

            Pro: trying the new magic thing with someone his own age for backup instead of facing down a bunch of forty-year-olds who could create waterspouts and pitch railcars by himself.

            Con: money.

            Pro: feeling good tomorrow instead of feeling ouch tomorrow.

            The cons aren’t really cons.

            He didn’t want to be shy about the tattoo forever with the people he actually lived with. And as for the money…

            Alden had given half of it to Boe so that his family and friends would be taken care of no matter where he was. He’d be spending half of what he had left on supplies, gear, and calling fees to make himself the most absurdly well-equipped and well-connected Rabbit on the Triplanets the next time he was summoned.

            A little would be set aside in his backup backup emergency fund.

            But all the rest of it…

            He’d just about talked himself into spending a million dollars however he wanted. As fast as he felt like. When the mood struck him.

            Neha had told him to save it for a downpayment on his future immortality, and everything on the internet was about preparing for retirement. Meanwhile, the tiny remnants of the person he’d been in January kept shouting that this much money should last him until he was a wrinkly old man.

            He thought all of them were overconfident in his ability to make it to the wrinkly stage of life.

            Besides, even if he did live to a ripe old age, two or three short summonings a year would make for a decent salary by his standards. He’d be absolutely floored if he only got summoned twice a year.

            Mind made up, he flicked through his interface and booked a recovery sauna slot. Then he quickly checked the North of North website to make sure he actually understood what you were supposed to do in potion saunas.

            Just sit there. No clothes or shoes. Towels. Shower first. No noisy, bright, or smelly spells. No food. Respect the tranquility of the environment.

            Gee they don’t tell you where to hide your auriad. That’s an oversight.

            He could go with under the wrist cuff or on a thigh. The auriad was getting more and more helpful every day about staying put comfortably wherever he wanted it to.

            A couple of minutes later, towel around his waist, he was stepping into the sauna.

            It was hot and spacious with soft lighting and steam. Five women, three men, and Haoyu each had plenty of room to themselves on the two tiers of pale wood benches. Alden assumed it was all normal enough…for Apex anyway. The adults mostly had that fit-and-flawless look about them that was so common in this gym he hardly ever noticed it anymore. And there was a little bitty cauldron on a pedestal in the middle of the floor. The clear substance inside was blub-blubbing, and he thought it was the source of the watermelonish scent.

            Haoyu smiled brightly and waved him over. “I’m going to watch all the drone footage of my rescues first,” he said quietly. “And then we have that reading assignment on ‘outsmarting enemies’ for offense.”

            Alden sat down beside him. By the time he’d started to sweat, he’d gotten over the inherent oddness of doing homework in such an uncommon environment—mostly because Haoyu was a dedicated student. Alden couldn’t stay hung up on towel etiquette while they were having a serious texting conversation about whether saving two of the sandbags at once would increase rescue speed enough to justify the difficulty, risk, and/or talent strain.

            The time went by quickly. A gym employee brought water for them and a refill potion for the tiny cauldron just as he finished up the reading for Instructor Klein. It had already been an hour. His homework was half done. He felt tired, but a little less so than when he’d sat down. Either his body liked saunas or it liked inhaling magic steam.

            Most of the people who’d been here to start with were gone. There was just one woman wearing cucumbers over her eyes while she lounged on the top bench…which was technically having food in the sauna, wasn’t it?

            He glanced over at Haoyu. His dark hair was stuck to his forehead, and his mouth was moving slightly. He was one of those people who did that even when they were reading silently.

            Alden kept at his own homework.

            Cucumber lady left about fifteen minutes later. Alden was busy watching a video, so he wouldn’t have noticed. But as soon as she closed the door behind her, Haoyu said, “Why can she wear food, but we can’t eat food?”

            “I was thinking the same thing.”

            Haoyu looked over at him. “Thanks for studying with me. I didn’t beg too much, did I? I know it’s expensive. I just got excited to have a gym buddy. The only other people I know of who are even close to our age here are a couple of uni first years.”

            “This is fun. I probably wouldn’t ever have tried it if you hadn’t asked, so I’m glad you did,” Alden said. “I was more worried about wearing my towel backwards than the money.”

            Haoyu looked thoughtful. “I don’t think you can wear a towel backward.”

            “What if I got here and everyone else had turned theirs into a toga? Or they were supposed to be for our heads instead of our waists?”

            Haoyu snorted.

            “It could happen. You Anesidorans have weird customs. Spas have weird customs. Magic has weird customs. All three combined in one location? If I’d walked in and found out we all had to remove our towels and bow to the cauldron, I wouldn’t have been very surprised.”

            “You were supposed to do that.”

            Alden stared at him.

            The other boy’s expression was sincere. “It’s all right that you didn’t do it. Since it was your first time, nobody expected you to know—”

            “Lexi’s right about you! You do deliver jokes in the exact same tone as serious material.”

            Haoyu grinned.

            “What if you do that to someone…you can’t do that! Someone like Jeffy will totally get naked and worship a wizard soup pot!”

            “Jokes are funnier when they’re subtle.” He lifted an arm up and over his head to start a shoulder stretch. “I’m glad you wanted to try it. I was a lot more focused knowing someone else was doing the same assignments.”

            Haoyu’s not going to mention it. Alden was gratified but not really surprised. He’s so good-natured. And his mom…

            Alden decided he wasn’t going to get a better chance to bring it up himself. “Do you think our classmates will make a big deal about the tattoo when they see it?”

            Arm still bent behind his head, Haoyu cut his eyes toward him. “It’s real, right?”

            “Yes.”

            “You just have to ignore them,” Haoyu said. “You’re rooming with me and Lexi and Lute… Plus everyone’s already talking about how something crazy happened to you. And you’re a Rabbit. The purity people have already slotted you in with the rest of us. They’re stupid and they just repeat whatever their parents say, so who cares?”

            Alden felt like he was missing at least a couple of social factors that Haoyu was treating as obvious. “The purity people?”

            “You know, the ones who are like, ‘Anyone who wants to work for the Artonans is an alien bumkisser. Anyone who gets summoned all the time cares more about money than human pride.’ Those people. I think there are only one or two in our whole class who are seriously that way.”

            He dropped his arm and shrugged.

            “I haven’t noticed,” said Alden.

            “Sometimes it’s someone who’s lost a family member to an emergency summons, and then, I understand. They’re grieving and angry. But usually it’s just the kids of Contract refusers, or people who’ve never been called at all for some reason, and they want to act like not being a part of that aspect of Avowed life somehow makes them better than everyone else.”

            “So they’re assholes?”

            “Mostly. Those people are going to be terrible no matter what, so I say don’t worry about them. Everyone else…”

            Haoyu’s brows suddenly lowered. “Maybe I wouldn’t flash it around the locker room, after all? Even if most of them don’t really have a problem with private contracts, they’re still going to say stupid things and gossip about it. It’s the same reason I don’t tell most people when my parents are doing something dangerous.”

            He nodded once, as if to settle the matter, then added, “And obviously don’t worry about me. Or any mature people. Mom and dad both have tattoos. From Triplanets work and from the demon suppression. Everyone who helps with that gets a secrecy one. And Lexi’s little sister has eight now!”

            “Half of them are turtles,” said Alden.

            Haoyu laughed. “Lexi won’t admit it, but he’s super worried that his parents are wrong and they won’t actually wash off. I think he’s already planning to murder anyone who says anything about the fact that Irina is blue.”

            ******

            They headed back to the dorm in the dark, and they met a rolling cafeteria delivery drone on the way that had half of Haoyu’s supper in it.

            “Listen,” Haoyu said, walking backward while he pleaded with the slowly trundling box, “that’s my vegetable rice. I’m Haoyu Zhang-Demir, Garden Hall, Suite 208. That’s me! My name’s on your lid display. Give it to me, and I’ll get to eat it sooner and you’ll get to go back to the cafeteria sooner. It makes sense!”

            Beeeeep, the drone said angrily when Haoyu moved to block its way again.

            “Not beep! Rice. My rice. I want it!”

            Alden was trying not to lose it. “I don’t think it understands you. It probably can’t give it to you until it reaches the room.”

            “It’s such a slow, dumb one! Why does CNH have slow, dumb drones?”

            Alden coughed. “You want me to pick it up?”

            “They make a huge racket and report you to drone operations if you interfere with their delivery for more than a second or two, and then an operator scolds you,” said Haoyu, stepping out of the drone’s path and rolling his eyes.

            “No, I mean…tell me to pick it up.”

            Haoyu blinked. “You mean with your skill?”

            “I recover from fatigue quickly, so it will work for this. And the drone can’t beep or report if it’s preserved, right? We’re just going to help it get to our room faster.”

            “That’s crazy. Drone operations is going to think it learned to teleport…let’s do it! Pick it up!”

            Alden swooped down on the drone.

            A few minutes later, laughing like dumbasses, they set the rolling box down in front of the doors to Garden Hall then hid behind a planter watching it.

            “It’s so freaked out,” Haoyu whispered, watching the box sit there silently.

            “How did I get here?” Alden whispered back. “Where am I? I am full of rice and confusion.”

            “You’re so weird.”

            “Shut up. You’re hiding behind a pepper plant with me.”

            “Peppers! Are these ones we can take? Oh, it’s figured it out. I’m getting a delivery notice.”

            He bounced over to the drone, and it’s lid popped open.

            “You two look happy,” Lute said when they stepped into the room shortly after that. He was lying on the chesterfield, reading one of Haoyu’s Chinese comic books. “Downright vibrant compared to Lexi. He staggered in, ate some of the food in the cooker, and he’s been in the tub ever since. Does he bathe with his whip?”

            “Maybe. He’s a Meister. They do things like that,” said Haoyu, setting his container of rice on the counter and lifting the lid on the chicken-onion dish he’d concocted. “This doesn’t look bad!”

            “It’s really good with ramen,” said Lute.

            “It is?! I have to call my mom!” Alden told himself he was content with the liquid salad he’d bought on the way out of the gym and the bean burgers he was microwaving. He carried his plate into the living room and took Lexi’s favorite armchair.

            “The self-mastery wordchain was fantastic,” he told Lute between bites of burger.

            Lute dropped the comic book on the ottoman and rolled over to look at him. “It’s cool, right?”

            “I don’t think I’d want to use it every day, since this shirt now has a hole burned into the back of it that seemed like a completely reasonable thing to do at the time. But it was so good in the gym. It ran out about the time Big Snake started pelting me with deathballs.”

            “Deathballs?”

            “Tennis balls at deadly speeds!” Haoyu called.

            “I know I was only my normal self again, and my normal self is pretty good at running around and ducking. But I felt so awkward all of the sudden. Like my limbs got stupider.”

            “Yeah, I’m always disappointed in my natural state of being when it fizzles out, too. Do you still want to learn it?”

            “I need to pay the debt. Can we do that tonight?”

            Lute looked at the plate on Alden’s lap. “Sure. But not while you’re eating. I bit my own finger doing that once.”

            Haoyu sat down with them a minute later, with a pile of food he’d very obviously tried to make photogenic. “We’ve done it. Our first official day of school.”

            “You cute little first years,” said Lute.

            “You’re a first year, too!” said Haoyu.

            “Yeah, but I’m a grizzled old veteran first year. I bet you two don’t even know which of the toilets on campus have spells and gag flushes on them.”

            Haoyu and Alden exchanged looks.

            “I feel like we need more details,” said Alden.

            “Absolutely not. It’s a right of passage to figure it out yourself.” He looked at Alden’s plate again. “You finished your burgers. How much fun can Haoyu and I have at your expense?”

            “What do you mean?”

            “As a friend, I should tell you to go get in bed for the night before I drop the debt on you. But…as a friend…I think you should let us observe you a while instead.”

            “Is he going to be funny?”

            “He’s going to be so funny.”

            Lute was grinning. Haoyu looked excited.

            ******

            Alden Thorn was pretty sure he’d done it. He’d just become a great roommate.

            Would a not great roommate let two teenage boys who were laughing like hyenas watch him try to wash dishes when he was in this state?

            He didn’t think so.

            The feet are more important, he told himself. Pay attention to your balance.

            He stood firm.

            The cup he’d been washing fell out of his hand into the small sink and sent soapy water splashing onto the floor. For the third time.

            “What the hell?”

            It came out sounding like “Whadell?”

            His tongue now had a disturbing habit of skipping over sounds. He accessed their roommate group chat.

            [Alden: Lute, did you somehow give me a triple dose of this shit!?]

            Haoyu was on the floor mopping spills with a kitchen towel they only owned thanks to his mother and her assistant explaining that kitchen towels were a must-have item. Lute was lying on his back on the table, clutching his stomach. They were both howling.

            [Alden: I can’t even talk! This can’t be normal.]

            It wasn’t exactly that he couldn’t talk. With careful effort, he could talk, stand up safely, or move his hands the way he wanted to. But doing all three at once was suddenly rocket science.

            Lute had even insisted on checking the water temperature before he put his hands in the sink. Apparently with this half of the wordchain, you could be minimally aware of the fact that you were boiling yourself.

            “I p-promise it’s only one!” Lute gasped. “After you finish the dishes, let’s…let’s try brushing your teeth!”

            [Alden: If there’s a fire tonight, I’ll die. I’ll never make it down the stairs.]

            “We’d carry you!” Haoyu said from the floor as another stream of bubbles slopped over the edge toward him.

            Lexi, scowling, stepped into the kitchen right then. He was in pajama pants. He crossed his arms over his bare chest. “Why are you three so loud, and what did you do to—?”

            “Hi,” said Alden. “I’m fine.”

            His arm overshot the drainboard by a mile, and a fork clattered across the tile counter.

            “Don’t fall!” Haoyu shouted.

            I’m not falling, thought Alden.

            But Haoyu was on his feet suddenly and Alden was being caught and set upright, so he must have been falling.

            Lexi walked over to the sink, his eyes wide. “Are you drunk?” he hissed in Alden’s ear. “Did you two get him drunk?!”

            “Yeah that’s it,” said Lute. “Just look at all this booze we’ve got lying around.”

            “I’mnah drunk! I’m jethnah master now.”

            Probably “I’m just not the master of myself right now” was an overly ambitious sentence.

            “He is drunk! He looks just like a drunk person on television! He sounds just like a drunk person.”

            “He’s not drunk,” said Haoyu, trying to square Alden’s shoulders from behind like he was straightening an unruly shelf.

            “Are niffing?” Alden asked as Lexi leaned toward him and inhaled.

            [Alden: He’s sniffing me. Stop it, Lexi. I don’t have beer breath. ]

            Lute started cackling again.

            “It’s a clumsiness wordchain,” Haoyu said. “That’s why he sounds fine on the room chat but not with his mouth. It seems pretty strong. Lute gave it to him.”

            “Pretty strong?” Lexi asked, staring as another tricky fork made a bid for freedom.

            “We aren’t letting him wash knives.”

            Lexi frowned. “Should you be letting him stand up?”

            [Alden: I’m a good roommate. I’m washing all the dishes by myself!]

            “Haoyu’s keeping you upright.”

            [Alden: He’s a good roommate, too.]

            [Haoyu: Thank you! That’s not a dish you’re washing though. It’s the pear you knocked in the sink earlier.]

            Alden looked down to see himself scrubbing the peel off a green pear with the dish brush. He set the fruit aside with as much dignity as he could.

            Lexi looked at them all, then he sighed and went to fetch the forks.

            After the dishes, they all three followed Alden around while he attempted to go through his nighttime routine. Equal amounts of teasing and mother-henning were involved. They wouldn’t let him trip over the edges of the rugs—maybe he shouldn’t have bought quite such thick ones—and bust his face. But they absolutely all stood there snickering while he tried to get toothpaste out of the tube onto his brush.

            “Don’t waste it!” said Haoyu as Alden stared down at the tablespoonful of green paste oozing down the sides of the brush onto his hand. “Put it all in your mouth.”

            “It’s not a mistake, you guys!” Lute said. “He wanted that much. You should have seen what he ate for lunch.”

            “Anesidoran social dynamic,” said Lexi. “Spit goes in the sink.”

            [Alden: Et tu, Lexi?]

            “Why do we randomly say ‘Anesidoran social dynamic’ to Alden sometimes?” Lute asked as he finished brushing.

            Alden glanced at them all in the mirror.

            Lexi’s expression was neutral.

            [Alden: Because I whined about Lexi keeping Anesidoran social dynamics to himself when he knew I was ignorant. I was being a dick.]

            About Lute. He felt even worse about that now.

            “Compared to Lexi?” Lute asked. .

            [Alden: That time I was.]

            Lute gasped in mock horror. “Cool! Anesidoran social dynamic—remember you have to pee before bed. For real.”

            Lexi and Haoyu both looked startled.

            “What kind of wordchain is this?” Lexi asked for the third time.

            ******

            Alden wasn’t sure about the lack of bodily awareness being great for sleeping. He might just have been tired after the day he’d had. But he was drifting off easily, right on the brink of unconsciousness, when the notification came in:

            [Video call from Twenty-seven Hundred and Sixty-third General Evul-art’h, Artona I. Connection fee waived.]

            Alden stared at it.

            Stuart was calling. Early. It shouldn’t have been much of a surprise. Alden was up in the middle of the night a lot, and Stuart knew that.

            But I’m in bed. And my tongue is broken.

            He reached up and slapped at the switch for the reading light over his bunk. When he finally got it on, he accepted the call with a thought.

            Evul-art’h’s black hair and pale pink eyes appeared. She was in her cushioned window seat again.

            “The human is very <<loosey-goosey>>today, Stu,” she reported. “He’s in bed.”

            Loosey-goosey? Alden was going to have to look into the word that was being translated that way.

            “Did we wake him up?” Stuart inserted his face between his sister and the tablet. “Did we wake you up? I’m sorry. I called as soon as I got home. I wasn’t thinking about the time as much as I should have been.”

            Evul-art’h gave them both a curious look.

            “It’s all right,” Alden said slowly in the least loosey-goosey Artonan he could manage. Think only about the tongue. Only the tongue. “I’m on the bad half of a wordchain now. Do you mind if we send text messages instead? Or talk in the morning? My mouth—”

            Stuart smiled. “Go to sleep. I’ll call back later.”

            “He means I’ll call back later,” his sister said dryly.

            “Thank you. Both of you. We can talk all day tomorrow, Stuart. If you want. Night-night.”

            This tale has been unlawfully lifted without the author's consent. Report any appearances on Amazon.

            “The human just said night-night and hung up on you. And me,” said Evul-art’h in an intrigued voice. “Mostly me, since it’s officially me calling.”

            “He has a name.” Stu-art’h perched on the edge of the lounger beside her.

            “I know. It’s your fault for naming your pet after him. Now I don’t know what to call either of them!”

            “Alden. And…Other Alden.”

            “The Ryeh-b’t and the other ryeh-b’t. Not confusing in the slightest.” She shoved herself back into her cushions. After a moment, she said, “Stu, are you all right these days?”

            He sighed. “I would be much more all right if you would all stop wanting things for me that I don’t want for myself.”

            She smiled. “You’re still angry with me. I don’t know why you thought I wouldn’t side with the others. It’s only the small matters I allow myself to be careless about. You are not a small matter.”

            One of her hands rummaged under a golden yellow pillow and pulled out one of her lungsticks. She held the dark tube out to him, and he lit it with a flick of his fingers.

            “You’ve always been such a beautiful caster, Stu.”

            He stood.

            “Thank you for calling Alden,” he said stiffly. “I would appreciate it if you did me the favor again when he has finished resting.”

            “I’ll do all manner of favors for you, baby brother.” She drew on the stick and sent a cloud of blue smoke toward the ceiling. “Consider doing just one for me in return.”

            ================================================ FILE: test/test-pages/royal-road/source.html ================================================ ONE HUNDRED TWO: What kind of wordchain? - Super Supportive | Royal Road

            Super Supportive

            by

            Sleyca

            ONE HUNDRED TWO: What kind of wordchain?

            Fiction Page
            Report Chapter
            Previous
            Chapter
            Next
            Chapter
            Advertisement
            Remove
             
            102
             
             
            “Were you expecting the competition for the showers to be the highest drama part of gym class?” Alden asked Haoyu as the two of them headed down a broad avenue toward North of North gym. It was a ten minute walk from the MPE building, right on the edge of campus, and apparently, they had both had the same idea when they discovered they were going to have to wait for their turns anyway.
            Haoyu was gulping from a water bottle. “I’m glad my group was getting along well. I think the one with the speedsters in it had some kind of issue. They seemed tense.”
            Finlay had looked pretty mad. It was the first time Alden had seen the Scottish boy in anything but a cheerful mood.
            “I didn’t have the attention span to watch them and run from Snake’s tennis ball pitches. It somehow adds insult to injury that he makes you chase the balls down after they hit you.”
            When they reached the airy glass building that Alden thought of as the main gym, the same girl in lime green yoga pants who’d been protecting the entrance from riffraff and gawkers on his first visit greeted them both warmly. She was giving out cups of some of the liquefied salads that the spa upstairs bottled and sold as drinks.
            They both took one.
            As they headed toward the showers, Haoyu sipped the black pepper beet juice he’d chosen and wrinkled his nose. “I don’t think I’m old enough to drink this.”
            Alden raised an eyebrow at him.
            “I’m trying!” Haoyu said. “Being friends with Lexi’s hard. He got really serious about nutrition in the past year or two. I saw him cut a single piece of chocolate in half once. And there I was stuffing six in my mouth at a time. I want to eat healthy, but I like candy so much. And anything fried.”
            “Is that why you bought the slow cooker?”
            Haoyu nodded. “That, and because my mom kept telling me I shouldn’t buy it because I wouldn’t ever use it. I’m going to use it every day. And send her pictures. Starting tonight. The bouncing oatmeal was just a temporary setback. It didn’t even taste that bad.”
            He looked curiously at a man casting spells on a section of the climbing wall as they passed it. “I’m so glad you’ve got a membership here, too. I really wanted one, and my parents wanted me to have it. But I almost told them not to buy it for me because it’s so…um…”
            “Richy?”
            “Yes. Mom and Dad both come here, but they have to. They need the facilities in the heavy gym. I could probably have used the school equipment until I was halfway through uni. But everything here is so much nicer, and I used to play on the rock wall in the onsite daycare and just imagine how great it was going to be when I was finally fifteen and I could use the real one. They can set it to have high winds, floods, and earthquakes! I went for it. And you’re here, too! We’ll both look richy together, and it won’t be as awkward when we leave school to use the better stuff.”
            “Happy to spoil myself with you.”
            They walked past an attendant into the room with the big rainfall showers and the ridiculously fluffy scented towels.
            Haoyu beamed at him. “Soooo, since you’re here with me, and you’re not going to freak out about it…I’ll confess that I actually already have potion therapy sauna slots booked for a few days a week until the end of term. I was always planning to come here instead of using the student showers.”
            Alden blinked at him. The potions saunas had been a part of his facilities tour. For a hundred argold—or more depending on which sauna you chose—you could sweat and inhale magical substances that did rather out-there sounding things.
            “Are you having your spirit waxed or…?”
            “No!” Haoyu laughed. “Is that even real? I’m using the recovery sauna. Potions aren’t as good as healers, obviously, but if I sit in there for ninety minutes or so, I don’t really need to rest on my rest days, and I don’t get too sore. I’m just going to do my homework in there instead of sitting through study halls. My mom did sauna homework her whole last year of uni. It was her idea, and since I wasn’t saying no to the gym membership I wasn’t going to say no to something like this.”
            He’s basically turning the two hour study hall into recovery time. Alden wondered if Mrs. Zhang-Demir had been attempting to balance out Haoyu’s lifestyle when she tried to set a dorm decorating budget.
            “That’s very efficient,” said Alden.
            “I’m telling you so you can book a slot tonight, if you want to do homework together. And so you’re not like, ‘Where did Haoyu go? Is he trying to get away from me?’ when I jump straight out of the shower and run off to the sauna instead of heading back to the dorms with you.”
            “Gotcha.”
            “Do you want to come?”
            “…isn’t it a lot of famous people sitting around in towels together, talking about superhero stuff?”
            That sounded like a high pressure environment to do homework in unless your parents were famous superheroes and you were used to them.
            “I’ve only used it a few times so far. It’s not that many people, and they’re all zoned out watching videos or working on their interfaces. They don’t talk and interrupt my studying, so it’s good.”
            “Let me think about it.”
            ******
            Alden had a couple of minutes to debate it while they both rinsed off.
            Pro: doing homework with Haoyu was in line with his mission to be a great roommate.
            Con: shirtlessness.
            Pro: trying the new magic thing with someone his own age for backup instead of facing down a bunch of forty-year-olds who could create waterspouts and pitch railcars by himself.
            Con: money.
            Pro: feeling good tomorrow instead of feeling ouch tomorrow.
            The cons aren’t really cons.
            He didn’t want to be shy about the tattoo forever with the people he actually lived with. And as for the money…
            Alden had given half of it to Boe so that his family and friends would be taken care of no matter where he was. He’d be spending half of what he had left on supplies, gear, and calling fees to make himself the most absurdly well-equipped and well-connected Rabbit on the Triplanets the next time he was summoned.
            A little would be set aside in his backup backup emergency fund.
            But all the rest of it…
            He’d just about talked himself into spending a million dollars however he wanted. As fast as he felt like. When the mood struck him.
            Neha had told him to save it for a downpayment on his future immortality, and everything on the internet was about preparing for retirement. Meanwhile, the tiny remnants of the person he’d been in January kept shouting that this much money should last him until he was a wrinkly old man.
            He thought all of them were overconfident in his ability to make it to the wrinkly stage of life.
            Besides, even if he did live to a ripe old age, two or three short summonings a year would make for a decent salary by his standards. He’d be absolutely floored if he only got summoned twice a year.
            Mind made up, he flicked through his interface and booked a recovery sauna slot. Then he quickly checked the North of North website to make sure he actually understood what you were supposed to do in potion saunas.
            Just sit there. No clothes or shoes. Towels. Shower first. No noisy, bright, or smelly spells. No food. Respect the tranquility of the environment.
            Gee they don’t tell you where to hide your auriad. That’s an oversight.
            He could go with under the wrist cuff or on a thigh. The auriad was getting more and more helpful every day about staying put comfortably wherever he wanted it to.
            A couple of minutes later, towel around his waist, he was stepping into the sauna.
            It was hot and spacious with soft lighting and steam. Five women, three men, and Haoyu each had plenty of room to themselves on the two tiers of pale wood benches. Alden assumed it was all normal enough…for Apex anyway. The adults mostly had that fit-and-flawless look about them that was so common in this gym he hardly ever noticed it anymore. And there was a little bitty cauldron on a pedestal in the middle of the floor. The clear substance inside was blub-blubbing, and he thought it was the source of the watermelonish scent.
            Haoyu smiled brightly and waved him over. “I’m going to watch all the drone footage of my rescues first,” he said quietly. “And then we have that reading assignment on ‘outsmarting enemies’ for offense.”
            Alden sat down beside him. By the time he’d started to sweat, he’d gotten over the inherent oddness of doing homework in such an uncommon environment—mostly because Haoyu was a dedicated student. Alden couldn’t stay hung up on towel etiquette while they were having a serious texting conversation about whether saving two of the sandbags at once would increase rescue speed enough to justify the difficulty, risk, and/or talent strain.
            The time went by quickly. A gym employee brought water for them and a refill potion for the tiny cauldron just as he finished up the reading for Instructor Klein. It had already been an hour. His homework was half done. He felt tired, but a little less so than when he’d sat down. Either his body liked saunas or it liked inhaling magic steam.
            Most of the people who’d been here to start with were gone. There was just one woman wearing cucumbers over her eyes while she lounged on the top bench…which was technically having food in the sauna, wasn’t it?
            He glanced over at Haoyu. His dark hair was stuck to his forehead, and his mouth was moving slightly. He was one of those people who did that even when they were reading silently.
            Alden kept at his own homework.
            Cucumber lady left about fifteen minutes later. Alden was busy watching a video, so he wouldn’t have noticed. But as soon as she closed the door behind her, Haoyu said, “Why can she wear food, but we can’t eat food?”
            “I was thinking the same thing.”
            Haoyu looked over at him. “Thanks for studying with me. I didn’t beg too much, did I? I know it’s expensive. I just got excited to have a gym buddy. The only other people I know of who are even close to our age here are a couple of uni first years.”
            “This is fun. I probably wouldn’t ever have tried it if you hadn’t asked, so I’m glad you did,” Alden said. “I was more worried about wearing my towel backwards than the money.”
            Haoyu looked thoughtful. “I don’t think you can wear a towel backward.”
            “What if I got here and everyone else had turned theirs into a toga? Or they were supposed to be for our heads instead of our waists?”
            Haoyu snorted.
            “It could happen. You Anesidorans have weird customs. Spas have weird customs. Magic has weird customs. All three combined in one location? If I’d walked in and found out we all had to remove our towels and bow to the cauldron, I wouldn’t have been very surprised.”
            “You were supposed to do that.”
            Alden stared at him.
            The other boy’s expression was sincere. “It’s all right that you didn’t do it. Since it was your first time, nobody expected you to know—”
            “Lexi’s right about you! You do deliver jokes in the exact same tone as serious material.”
            Haoyu grinned.
            “What if you do that to someone…you can’t do that! Someone like Jeffy will totally get naked and worship a wizard soup pot!”
            “Jokes are funnier when they’re subtle.” He lifted an arm up and over his head to start a shoulder stretch. “I’m glad you wanted to try it. I was a lot more focused knowing someone else was doing the same assignments.”
            Haoyu’s not going to mention it. Alden was gratified but not really surprised. He’s so good-natured. And his mom…
            Alden decided he wasn’t going to get a better chance to bring it up himself. “Do you think our classmates will make a big deal about the tattoo when they see it?”
            Arm still bent behind his head, Haoyu cut his eyes toward him. “It’s real, right?”
            “Yes.”
            “You just have to ignore them,” Haoyu said. “You’re rooming with me and Lexi and Lute… Plus everyone’s already talking about how something crazy happened to you. And you’re a Rabbit. The purity people have already slotted you in with the rest of us. They’re stupid and they just repeat whatever their parents say, so who cares?”
            Alden felt like he was missing at least a couple of social factors that Haoyu was treating as obvious. “The purity people?”
            “You know, the ones who are like, ‘Anyone who wants to work for the Artonans is an alien bumkisser. Anyone who gets summoned all the time cares more about money than human pride.’ Those people. I think there are only one or two in our whole class who are seriously that way.”
            He dropped his arm and shrugged.
            “I haven’t noticed,” said Alden.
            “Sometimes it’s someone who’s lost a family member to an emergency summons, and then, I understand. They’re grieving and angry. But usually it’s just the kids of Contract refusers, or people who’ve never been called at all for some reason, and they want to act like not being a part of that aspect of Avowed life somehow makes them better than everyone else.”
            “So they’re assholes?”
            “Mostly. Those people are going to be terrible no matter what, so I say don’t worry about them. Everyone else…”
            Haoyu’s brows suddenly lowered. “Maybe I wouldn’t flash it around the locker room, after all? Even if most of them don’t really have a problem with private contracts, they’re still going to say stupid things and gossip about it. It’s the same reason I don’t tell most people when my parents are doing something dangerous.”
            He nodded once, as if to settle the matter, then added, “And obviously don’t worry about me. Or any mature people. Mom and dad both have tattoos. From Triplanets work and from the demon suppression. Everyone who helps with that gets a secrecy one. And Lexi’s little sister has eight now!”
            “Half of them are turtles,” said Alden.
            Haoyu laughed. “Lexi won’t admit it, but he’s super worried that his parents are wrong and they won’t actually wash off. I think he’s already planning to murder anyone who says anything about the fact that Irina is blue.”
            ******
            They headed back to the dorm in the dark, and they met a rolling cafeteria delivery drone on the way that had half of Haoyu’s supper in it.
            “Listen,” Haoyu said, walking backward while he pleaded with the slowly trundling box, “that’s my vegetable rice. I’m Haoyu Zhang-Demir, Garden Hall, Suite 208. That’s me! My name’s on your lid display. Give it to me, and I’ll get to eat it sooner and you’ll get to go back to the cafeteria sooner. It makes sense!”
            Beeeeep, the drone said angrily when Haoyu moved to block its way again.
            “Not beep! Rice. My rice. I want it!”
            Alden was trying not to lose it. “I don’t think it understands you. It probably can’t give it to you until it reaches the room.”
            “It’s such a slow, dumb one! Why does CNH have slow, dumb drones?”
            Alden coughed. “You want me to pick it up?”
            “They make a huge racket and report you to drone operations if you interfere with their delivery for more than a second or two, and then an operator scolds you,” said Haoyu, stepping out of the drone’s path and rolling his eyes.
            “No, I mean…tell me to pick it up.”
            Haoyu blinked. “You mean with your skill?”
            “I recover from fatigue quickly, so it will work for this. And the drone can’t beep or report if it’s preserved, right? We’re just going to help it get to our room faster.”
            “That’s crazy. Drone operations is going to think it learned to teleport…let’s do it! Pick it up!”
            Alden swooped down on the drone.
            A few minutes later, laughing like dumbasses, they set the rolling box down in front of the doors to Garden Hall then hid behind a planter watching it.
            “It’s so freaked out,” Haoyu whispered, watching the box sit there silently.
            “How did I get here?” Alden whispered back. “Where am I? I am full of rice and confusion.”
            “You’re so weird.”
            “Shut up. You’re hiding behind a pepper plant with me.”
            “Peppers! Are these ones we can take? Oh, it’s figured it out. I’m getting a delivery notice.”
            He bounced over to the drone, and it’s lid popped open.
            “You two look happy,” Lute said when they stepped into the room shortly after that. He was lying on the chesterfield, reading one of Haoyu’s Chinese comic books. “Downright vibrant compared to Lexi. He staggered in, ate some of the food in the cooker, and he’s been in the tub ever since. Does he bathe with his whip?”
            “Maybe. He’s a Meister. They do things like that,” said Haoyu, setting his container of rice on the counter and lifting the lid on the chicken-onion dish he’d concocted. “This doesn’t look bad!”
            “It’s really good with ramen,” said Lute.
            “It is?! I have to call my mom!” Alden told himself he was content with the liquid salad he’d bought on the way out of the gym and the bean burgers he was microwaving. He carried his plate into the living room and took Lexi’s favorite armchair.
            “The self-mastery wordchain was fantastic,” he told Lute between bites of burger.
            Lute dropped the comic book on the ottoman and rolled over to look at him. “It’s cool, right?”
            “I don’t think I’d want to use it every day, since this shirt now has a hole burned into the back of it that seemed like a completely reasonable thing to do at the time. But it was so good in the gym. It ran out about the time Big Snake started pelting me with deathballs.”
            “Deathballs?”
            “Tennis balls at deadly speeds!” Haoyu called.
            “I know I was only my normal self again, and my normal self is pretty good at running around and ducking. But I felt so awkward all of the sudden. Like my limbs got stupider.”
            “Yeah, I’m always disappointed in my natural state of being when it fizzles out, too. Do you still want to learn it?”
            “I need to pay the debt. Can we do that tonight?”
            Lute looked at the plate on Alden’s lap. “Sure. But not while you’re eating. I bit my own finger doing that once.”
            Haoyu sat down with them a minute later, with a pile of food he’d very obviously tried to make photogenic. “We’ve done it. Our first official day of school.”
            “You cute little first years,” said Lute.
            “You’re a first year, too!” said Haoyu.
            “Yeah, but I’m a grizzled old veteran first year. I bet you two don’t even know which of the toilets on campus have spells and gag flushes on them.”
            Haoyu and Alden exchanged looks.
            “I feel like we need more details,” said Alden.
            “Absolutely not. It’s a right of passage to figure it out yourself.” He looked at Alden’s plate again. “You finished your burgers. How much fun can Haoyu and I have at your expense?”
            “What do you mean?”
            “As a friend, I should tell you to go get in bed for the night before I drop the debt on you. But…as a friend…I think you should let us observe you a while instead.”
            “Is he going to be funny?”
            “He’s going to be so funny.”
            Lute was grinning. Haoyu looked excited.
            ******
            Alden Thorn was pretty sure he’d done it. He’d just become a great roommate.
            Would a not great roommate let two teenage boys who were laughing like hyenas watch him try to wash dishes when he was in this state?
            He didn’t think so.
            The feet are more important, he told himself. Pay attention to your balance.
            He stood firm.
            The cup he’d been washing fell out of his hand into the small sink and sent soapy water splashing onto the floor. For the third time.
            “What the hell?”
            It came out sounding like “Whadell?”
            His tongue now had a disturbing habit of skipping over sounds. He accessed their roommate group chat.
            [Alden: Lute, did you somehow give me a triple dose of this shit!?]
            Haoyu was on the floor mopping spills with a kitchen towel they only owned thanks to his mother and her assistant explaining that kitchen towels were a must-have item. Lute was lying on his back on the table, clutching his stomach. They were both howling.
            [Alden: I can’t even talk! This can’t be normal.]
            It wasn’t exactly that he couldn’t talk. With careful effort, he could talk, stand up safely, or move his hands the way he wanted to. But doing all three at once was suddenly rocket science.
            Lute had even insisted on checking the water temperature before he put his hands in the sink. Apparently with this half of the wordchain, you could be minimally aware of the fact that you were boiling yourself.
            “I p-promise it’s only one!” Lute gasped. “After you finish the dishes, let’s…let’s try brushing your teeth!”
            [Alden: If there’s a fire tonight, I’ll die. I’ll never make it down the stairs.]
            “We’d carry you!” Haoyu said from the floor as another stream of bubbles slopped over the edge toward him.
            Lexi, scowling, stepped into the kitchen right then. He was in pajama pants. He crossed his arms over his bare chest. “Why are you three so loud, and what did you do to—?”
            “Hi,” said Alden. “I’m fine.”
            His arm overshot the drainboard by a mile, and a fork clattered across the tile counter.
            “Don’t fall!” Haoyu shouted.
            I’m not falling, thought Alden.
            But Haoyu was on his feet suddenly and Alden was being caught and set upright, so he must have been falling.
            Lexi walked over to the sink, his eyes wide. “Are you drunk?” he hissed in Alden’s ear. “Did you two get him drunk?!”
            “Yeah that’s it,” said Lute. “Just look at all this booze we’ve got lying around.”
            “I’mnah drunk! I’m jethnah master now.”
            Probably “I’m just not the master of myself right now” was an overly ambitious sentence.
            “He is drunk! He looks just like a drunk person on television! He sounds just like a drunk person.”
            “He’s not drunk,” said Haoyu, trying to square Alden’s shoulders from behind like he was straightening an unruly shelf.
            “Are niffing?” Alden asked as Lexi leaned toward him and inhaled.
            [Alden: He’s sniffing me. Stop it, Lexi. I don’t have beer breath. ]
            Lute started cackling again.
            “It’s a clumsiness wordchain,” Haoyu said. “That’s why he sounds fine on the room chat but not with his mouth. It seems pretty strong. Lute gave it to him.”
            “Pretty strong?” Lexi asked, staring as another tricky fork made a bid for freedom.
            “We aren’t letting him wash knives.”
            Lexi frowned. “Should you be letting him stand up?”
            [Alden: I’m a good roommate. I’m washing all the dishes by myself!]
            “Haoyu’s keeping you upright.”
            [Alden: He’s a good roommate, too.]
            [Haoyu: Thank you! That’s not a dish you’re washing though. It’s the pear you knocked in the sink earlier.]
            Alden looked down to see himself scrubbing the peel off a green pear with the dish brush. He set the fruit aside with as much dignity as he could.
            Lexi looked at them all, then he sighed and went to fetch the forks.
            After the dishes, they all three followed Alden around while he attempted to go through his nighttime routine. Equal amounts of teasing and mother-henning were involved. They wouldn’t let him trip over the edges of the rugs—maybe he shouldn’t have bought quite such thick ones—and bust his face. But they absolutely all stood there snickering while he tried to get toothpaste out of the tube onto his brush.
            “Don’t waste it!” said Haoyu as Alden stared down at the tablespoonful of green paste oozing down the sides of the brush onto his hand. “Put it all in your mouth.”
            “It’s not a mistake, you guys!” Lute said. “He wanted that much. You should have seen what he ate for lunch.”
            “Anesidoran social dynamic,” said Lexi. “Spit goes in the sink.”
            [Alden: Et tu, Lexi?]
            “Why do we randomly say ‘Anesidoran social dynamic’ to Alden sometimes?” Lute asked as he finished brushing.
            Alden glanced at them all in the mirror.
            Lexi’s expression was neutral.
            [Alden: Because I whined about Lexi keeping Anesidoran social dynamics to himself when he knew I was ignorant. I was being a dick.]
            About Lute. He felt even worse about that now.
            “Compared to Lexi?” Lute asked. .
            [Alden: That time I was.]
            Lute gasped in mock horror. “Cool! Anesidoran social dynamic—remember you have to pee before bed. For real.”
            Lexi and Haoyu both looked startled.
            “What kind of wordchain is this?” Lexi asked for the third time.
            ******
            Alden wasn’t sure about the lack of bodily awareness being great for sleeping. He might just have been tired after the day he’d had. But he was drifting off easily, right on the brink of unconsciousness, when the notification came in:
            [Video call from Twenty-seven Hundred and Sixty-third General Evul-art’h, Artona I. Connection fee waived.]
            Alden stared at it.
            Stuart was calling. Early. It shouldn’t have been much of a surprise. Alden was up in the middle of the night a lot, and Stuart knew that.
            But I’m in bed. And my tongue is broken.
            He reached up and slapped at the switch for the reading light over his bunk. When he finally got it on, he accepted the call with a thought.
            Evul-art’h’s black hair and pale pink eyes appeared. She was in her cushioned window seat again.
            “The human is very <<loosey-goosey>>today, Stu,” she reported. “He’s in bed.”
            Loosey-goosey? Alden was going to have to look into the word that was being translated that way.
            “Did we wake him up?” Stuart inserted his face between his sister and the tablet. “Did we wake you up? I’m sorry. I called as soon as I got home. I wasn’t thinking about the time as much as I should have been.”
            Evul-art’h gave them both a curious look.
            “It’s all right,” Alden said slowly in the least loosey-goosey Artonan he could manage. Think only about the tongue. Only the tongue. “I’m on the bad half of a wordchain now. Do you mind if we send text messages instead? Or talk in the morning? My mouth—”
            Stuart smiled. “Go to sleep. I’ll call back later.”
            “He means I’ll call back later,” his sister said dryly.
            “Thank you. Both of you. We can talk all day tomorrow, Stuart. If you want. Night-night.”
             

            This tale has been unlawfully lifted without the author's consent. Report any appearances on Amazon.

            “The human just said night-night and hung up on you. And me,” said Evul-art’h in an intrigued voice. “Mostly me, since it’s officially me calling.”
            “He has a name.” Stu-art’h perched on the edge of the lounger beside her.
            “I know. It’s your fault for naming your pet after him. Now I don’t know what to call either of them!”
            “Alden. And…Other Alden.”
            “The Ryeh-b’t and the other ryeh-b’t. Not confusing in the slightest.” She shoved herself back into her cushions. After a moment, she said, “Stu, are you all right these days?”
            He sighed. “I would be much more all right if you would all stop wanting things for me that I don’t want for myself.”
            She smiled. “You’re still angry with me. I don’t know why you thought I wouldn’t side with the others. It’s only the small matters I allow myself to be careless about. You are not a small matter.”
            One of her hands rummaged under a golden yellow pillow and pulled out one of her lungsticks. She held the dark tube out to him, and he lit it with a flick of his fingers.
            “You’ve always been such a beautiful caster, Stu.”
            He stood.
            “Thank you for calling Alden,” he said stiffly. “I would appreciate it if you did me the favor again when he has finished resting.”
            “I’ll do all manner of favors for you, baby brother.” She drew on the stick and sent a cloud of blue smoke toward the ceiling. “Consider doing just one for me in return.”
            Advertisement
            Remove
            A note from Sleyca

            *****Notes*****

            ** Not really a note, but do you guys know how different sauna and locker room etiquette can be from country to country...or in the US, at least, from school to school and gym to gym? And generational differences, too. It's one of those weird things I never think about, and then I have to all of the sudden for the story. **

            Other Alden -- the most important character that never gets screentime. Other Alden is a red female ryeh-b't. She looks like a cross between a dragon and a bipedal dinosaur. Only small. Stu-art'h bought her egg after First Alden's assumed death. He turned his bathroom into a terrarium-like habitat for her. She has her own little splashing pool in there. Many Artonan children train ryeh-b'ts for flight, message carrying, and hunting competitions; Stuart says he's giving OA the same training even though he doesn't plan to let her compete.

            Art'h -- The surname used by the Primary's family. It sounds almost like "art" with a nearly silent exhalation on the end. So technically there is a difference in "Stuart" and "Stu-art'h" as far as pronunciation goes.

            Support "Super Supportive "
            Patreon
            Previous
            Chapter             Fiction
            Index             Next
            Chapter
            About the author
            Sleyca
            1 Fictions 8 Posts 0 Threads

            Sleyca

            Bio: I write things. I hope you like them.

            Achievements
            Comments(301)
            Log in to comment
            Log In
            Log in to comment
            Log In

            About

            Royal Road® is the home of web novels and fan fictions! In our amazing community, you can find various talented individuals who write as a hobby or even professionally, artists who create art for them, and many, many readers who provide valuable feedback and encouragement.

            Amazon Affiliate

            Royal Road® as an Amazon Associate earns from qualifying purchases.


            Follow Us On

            Need some help?



            Advertising

            Royal Road® © 2024
            Terms of Service | Privacy Policy | Cookie Policy | DMCA | Blog | Status
            ================================================ FILE: test/test-pages/rtl-1/expected-metadata.json ================================================ { "title": "RTL Test", "byline": null, "dir": "rtl", "excerpt": "Lorem ipsum dolor sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/rtl-1/expected.html ================================================

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-1/source.html ================================================ RTL Test

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-2/expected-metadata.json ================================================ { "title": "RTL Test", "byline": null, "dir": "rtl", "excerpt": "Lorem ipsum dolor sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/rtl-2/expected.html ================================================

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-2/source.html ================================================ RTL Test

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-3/expected-metadata.json ================================================ { "title": "RTL Test", "byline": null, "dir": "rtl", "excerpt": "Lorem ipsum dolor sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/rtl-3/expected.html ================================================

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-3/source.html ================================================ RTL Test

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-4/expected-metadata.json ================================================ { "title": "RTL Test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/rtl-4/expected.html ================================================

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/rtl-4/source.html ================================================ RTL Test

            Lorem

            Lorem ipsum dolor sit amet.

            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

            ================================================ FILE: test/test-pages/salon-1/expected-metadata.json ================================================ { "title": "The sharing economy is a lie: Uber, Ayn Rand and the truth about tech and libertarians", "byline": "Joanna Rothkopf", "dir": null, "excerpt": "Disruptive companies talk a good game about sharing. Uber's really just an under-regulated company making riches", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/salon-1/expected.html ================================================

            Horror stories about the increasingly unpopular taxi service Uber have been commonplace in recent months, but there is still much to be learned from its handling of the recent hostage drama in downtown Sydney, Australia. We’re told that we reveal our true character in moments of crisis, and apparently that’s as true for companies as it is for individuals.

            A number of experts have challenged the idea that the horrific explosion of violence in a Sydney café was “terrorism,” since the attacker was mentally unbalanced and acted alone. But, terror or not, the ordeal was certainly terrifying. Amid the chaos and uncertainty, the city believed itself to be under a coordinated and deadly attack.

            Uber had an interesting, if predictable, response to the panic and mayhem: It raised prices. A lot.

            In case you missed the story, the facts are these: Someone named Man Haron Monis, who was considered mentally unstable and had been investigated for murdering his ex-wife, seized hostages in a café that was located in Sydney’s Central Business District or “CBD.” In the process he put up an Islamic flag – “igniting,” as Reuters reported, “fears of a jihadist attack in the heart of the country’s biggest city.”

            In the midst of the fear, Uber stepped in and tweeted this announcement:  “We are all concerned with events in CBD. Fares have increased to encourage more drivers to come online & pick up passengers in the area.”

            As Mashable reports, the company announced that it would charge a minimum of $100 Australian to take passengers from the area immediately surrounding the ongoing crisis, and prices increased by as much as four times the standard amount. A firestorm of criticism quickly erupted – “@Uber_Sydney stop being assholes,” one Twitter response began – and Uber soon found itself offering free rides out of the troubled area instead.

            That opener suggests that Uber, as part of a community under siege, is preparing to respond in a civic manner.

            “… Fares have increased to encourage more drivers to come online & pick up passengers in the area.”

            But, despite the expression of shared concern, there is no sense of civitas to be found in the statement that follows. There is only a transaction, executed at what the corporation believes to be market value. Lesson #1 about Uber is, therefore, that in its view there is no heroism, only self-interest. This is Ayn Rand’s brutal, irrational and primitive philosophy in its purest form: altruism is evil, and self-interest is the only true heroism.

            There was once a time when we might have read of “hero cabdrivers” or “hero bus drivers” placing themselves in harm’s way to rescue their fellow citizens. For its part, Uber might have suggested that it would use its network of drivers and its scheduling software to recruit volunteer drivers for a rescue mission.

            Instead, we are told that Uber’s pricing surge was its expression of concern. Uber’s way to address a human crisis is apparently by letting the market govern human behavior, as if there were (in libertarian economist Tyler Cowen’s phrase) “markets in everything” – including the lives of a city’s beleaguered citizens (and its Uber drivers).

            Where would this kind of market-driven practice leave poor or middle-income citizens in a time of crisis? If they can’t afford the “surged” price, apparently it would leave them squarely in the line of fire. And come to think of it, why would Uber drivers value their lives so cheaply, unless they’re underpaid?

            One of the lessons of Sydney is this: Uber’s philosophy, whether consciously expressed or not, is that life belongs to the highest bidder – and therefore, by implication, the highest bidder’s life has the greatest value. Society, on the other hand, may choose to believe that every life has equal value – or that lifesaving services should be available at affordable prices.

            If nothing else, the Sydney experience should prove once and for all that there is no such thing as “the sharing economy.” Uber is a taxi company, albeit an under-regulated one, and nothing more. It’s certainly not a “ride sharing” service, where someone who happens to be going in the same direction is willing to take along an extra passenger and split gas costs. A ride-sharing service wouldn’t find itself “increasing fares to encourage more drivers” to come into Sydney’s terrorized Central Business District.

            A “sharing economy,” by definition, is lateral in structure. It is a peer-to-peer economy. But Uber, as its name suggests, is hierarchical in structure. It monitors and controls its drivers, demanding that they purchase services from it while guiding their movements and determining their level of earnings. And its pricing mechanisms impose unpredictable costs on its customers, extracting greater amounts whenever the data suggests customers can be compelled to pay them.

            This is a top-down economy, not a “shared” one.

            A number of Uber’s fans and supporters defended the company on the grounds that its “surge prices,” including those seen during the Sydney crisis, are determined by an algorithm. But an algorithm can be an ideological statement, and is always a cultural artifact. As human creations, algorithms reflect their creators.

            Uber’s tweet during the Sydney crisis made it sound as if human intervention, rather than algorithmic processes, caused prices to soar that day. But it doesn’t really matter if that surge was manually or algorithmically driven. Either way the prices were Uber’s doing – and its moral choice.

            Uber has been strenuously defending its surge pricing in the wake of accusations (apparently justified) that the company enjoyed windfall profits during Hurricane Sandy. It has now promised the state of New York that it will cap its surge prices (at three times the highest rate on two non-emergency days). But if Uber has its way, it will soon enjoy a monopolistic stranglehold on car service rates in most major markets. And it has demonstrated its willingness to ignore rules and regulations. That means predictable and affordable taxi fares could become a thing of the past.

            In practice, surge pricing could become a new, privatized form of taxation on middle-class taxi customers.

            Even without surge pricing, Uber and its supporters are hiding its full costs. When middle-class workers are underpaid or deprived of benefits and full working rights, as Uber’s reportedly are, the entire middle-class economy suffers. Overall wages and benefits are suppressed for the majority, while the wealthy few are made even richer. The invisible costs of ventures like Uber are extracted over time, far surpassing whatever short-term savings they may occasionally offer.

            Like Walmart, Uber underpays its employees – many of its drivers are employees, in everything but name – and then drains the social safety net to make up the difference. While Uber preaches libertarianism, it practices a form of corporate welfare. It’s reportedly celebrating Obamacare, for example, since the Affordable Care Act allows it to avoid providing health insurance to its workforce. But the ACA’s subsidies, together with Uber’s often woefully insufficient wages, mean that the rest of us are paying its tab instead. And the lack of income security among Uber’s drivers creates another social cost for Americans – in lost tax revenue, and possibly in increased use of social services.

            The company’s war on regulation will also carry a social price. Uber and its supporters don’t seem to understand that regulations exist for a reason. It’s true that nobody likes excessive bureaucracy, but not all regulations are excessive or onerous. And when they are, it’s a flaw in execution rather than principle.

            Regulations were created because they serve a social purpose, ensuring the free and fair exchange of services and resources among all segments of society. Some services, such as transportation, are of such importance that the public has a vested interest in ensuring they will be readily available at reasonably affordable prices. That’s not unreasonable for taxi services, especially given the fact that they profit from publicly maintained roads and bridges.

            Uber has presented itself as a modernized, efficient alternative to government oversight. But it’s an evasion of regulation, not its replacement. As Alexis Madrigalreports, Uber has deliberately ignored city regulators and used customer demand to force its model of inadequate self-governance (my conclusion, not his) onto one city after another.

            Uber presented itself as a refreshing alternative to the over-bureaucratized world of urban transportation. But that’s a false choice. We can streamline sclerotic city regulators, upgrade taxi fleets and even provide users with fancy apps that make it easier to call a cab. The company’s binary presentation – us, or City Hall – frames the debate in artificial terms.

            Uber claims that its driver rating system is a more efficient way to monitor drivers, but that’s an entirely unproven assumption. While taxi drivers have been known to misbehave, the worldwide litany of complaints against Uber drivers – for everything from dirty cars and spider bites to assault with a hammer, fondling and rape– suggest that Uber’s system may not work as well as old-fashioned regulation. It’s certainly not noticeably superior.

            In fact, prosecutors in San Francisco and Los Angeles say Uber has been lying to its customers about the level and quality of its background checks. The company now promises it will do a better job at screening drivers. But it won’t tell us what measures its taking to improve its safety record, and it’s fighting the kind of driver scrutiny that taxicab companies have been required to enforce for many decades.

            Many reports suggest that beleaguered drivers don’t feel much better about the company than victimized passengers do. They tell horror stories about the company’s hiring and management practices. Uber unilaterally slashes drivers’ rates, while claiming they don’t need to unionize. (The Teamsters disagree.)

            The company also pushes sketchy, substandard loans onto its drivers – but hey, what could go wrong?

            Uber has many libertarian defenders. And yet, it deceives the press and threatens to spy on journalists, lies to its own employees, keeps its practices a secret and routinely invades the privacy of civilians – sometimes merely for entertainment. (It has a tool, with the Orwellian name the “God View,” that it can use for monitoring customers’ personal movements.)

            Aren’t those the kinds of things libertarians say they hate about government?

            This isn’t a “gotcha” exercise. It matters. Uber is the poster child for the pro-privatization, anti-regulatory ideology that ascribes magical powers to technology and the private sector. It is deeply a political entity, from its Nietzschean name to its recent hiring of White House veteran David Plouffe. Uber is built around a relatively simple app (which relies on government-created technology), but it’s not really a tech company. Above all else Uber is an ideological campaign, a neoliberal project whose real products are deregulation and the dismantling of the social contract.

            Or maybe, as that tweeter in Sydney suggested, they’re just assholes.

            Either way, it’s important that Uber’s worldview and business practices not be allowed to “disrupt” our economy or our social fabric. People who work hard deserve to make a decent living. Society at large deserves access to safe and affordable transportation. And government, as the collective expression of a democratic society, has a role to play in protecting its citizens.

            And then there’s the matter of our collective psyche. In her book “A Paradise Built in Hell: The Extraordinary Communities that Arise in Disaster,” Rebecca Solnit wrote of the purpose, meaning and deep satisfaction people find when they pull together to help one another in the face of adversity.  But in the world Uber seeks to create, those surges of the spirit would be replaced by surge pricing.

            You don’t need a “God view” to see what happens next. When heroism is reduced to a transaction, the soul of a society is sold cheap.

            ================================================ FILE: test/test-pages/salon-1/source.html ================================================ The sharing economy is a lie: Uber, Ayn Rand and the truth about tech and libertarians - Salon.com

            Salon

            Sunday, Feb 1, 2015 11:57 AM UTC

            The sharing economy is a lie: Uber, Ayn Rand and the truth about tech and libertarians

            Disruptive companies talk a good game about sharing. Uber's really just an under-regulated company making riches

            Topics: uber, libertarians, Sydney, Editor's Picks, sharing economy,

            The sharing economy is a lie: Uber, Ayn Rand and the truth about tech and libertarians Ayn Rand, Rand Paul (Credit: AP/Manuel Balce Ceneta/Photo montage by Salon)

            Horror stories about the increasingly unpopular taxi service Uber have been commonplace in recent months, but there is still much to be learned from its handling of the recent hostage drama in downtown Sydney, Australia. We’re told that we reveal our true character in moments of crisis, and apparently that’s as true for companies as it is for individuals.

            A number of experts have challenged the idea that the horrific explosion of violence in a Sydney café was “terrorism,” since the attacker was mentally unbalanced and acted alone. But, terror or not, the ordeal was certainly terrifying. Amid the chaos and uncertainty, the city believed itself to be under a coordinated and deadly attack.

            Uber had an interesting, if predictable, response to the panic and mayhem: It raised prices. A lot.

            In case you missed the story, the facts are these: Someone named Man Haron Monis, who was considered mentally unstable and had been investigated for murdering his ex-wife, seized hostages in a café that was located in Sydney’s Central Business District or “CBD.” In the process he put up an Islamic flag – “igniting,” as Reuters reported, “fears of a jihadist attack in the heart of the country’s biggest city.”

            In the midst of the fear, Uber stepped in and tweeted this announcement:  “We are all concerned with events in CBD. Fares have increased to encourage more drivers to come online & pick up passengers in the area.”

            As Mashable reports, the company announced that it would charge a minimum of $100 Australian to take passengers from the area immediately surrounding the ongoing crisis, and prices increased by as much as four times the standard amount. A firestorm of criticism quickly erupted – “@Uber_Sydney stop being assholes,” one Twitter response began – and Uber soon found itself offering free rides out of the troubled area instead.

            What can we learn from this incident? Let’s start by parsing that tweet:

            “We are all concerned with events in CBD …”

            That opener suggests that Uber, as part of a community under siege, is preparing to respond in a civic manner.

            “… Fares have increased to encourage more drivers to come online & pick up passengers in the area.”

            But, despite the expression of shared concern, there is no sense of civitas to be found in the statement that follows. There is only a transaction, executed at what the corporation believes to be market value. Lesson #1 about Uber is, therefore, that in its view there is no heroism, only self-interest. This is Ayn Rand’s brutal, irrational and primitive philosophy in its purest form: altruism is evil, and self-interest is the only true heroism.

            There was once a time when we might have read of “hero cabdrivers” or “hero bus drivers” placing themselves in harm’s way to rescue their fellow citizens. For its part, Uber might have suggested that it would use its network of drivers and its scheduling software to recruit volunteer drivers for a rescue mission.

            Instead, we are told that Uber’s pricing surge was its expression of concern. Uber’s way to address a human crisis is apparently by letting the market govern human behavior, as if there were (in libertarian economist Tyler Cowen’s phrase) “markets in everything” – including the lives of a city’s beleaguered citizens (and its Uber drivers).

            Where would this kind of market-driven practice leave poor or middle-income citizens in a time of crisis? If they can’t afford the “surged” price, apparently it would leave them squarely in the line of fire. And come to think of it, why would Uber drivers value their lives so cheaply, unless they’re underpaid?

            One of the lessons of Sydney is this: Uber’s philosophy, whether consciously expressed or not, is that life belongs to the highest bidder – and therefore, by implication, the highest bidder’s life has the greatest value. Society, on the other hand, may choose to believe that every life has equal value – or that lifesaving services should be available at affordable prices.

            If nothing else, the Sydney experience should prove once and for all that there is no such thing as “the sharing economy.” Uber is a taxi company, albeit an under-regulated one, and nothing more. It’s certainly not a “ride sharing” service, where someone who happens to be going in the same direction is willing to take along an extra passenger and split gas costs. A ride-sharing service wouldn’t find itself “increasing fares to encourage more drivers” to come into Sydney’s terrorized Central Business District.

            A “sharing economy,” by definition, is lateral in structure. It is a peer-to-peer economy. But Uber, as its name suggests, is hierarchical in structure. It monitors and controls its drivers, demanding that they purchase services from it while guiding their movements and determining their level of earnings. And its pricing mechanisms impose unpredictable costs on its customers, extracting greater amounts whenever the data suggests customers can be compelled to pay them.

            This is a top-down economy, not a “shared” one.

            A number of Uber’s fans and supporters defended the company on the grounds that its “surge prices,” including those seen during the Sydney crisis, are determined by an algorithm. But an algorithm can be an ideological statement, and is always a cultural artifact. As human creations, algorithms reflect their creators.

            Uber’s tweet during the Sydney crisis made it sound as if human intervention, rather than algorithmic processes, caused prices to soar that day. But it doesn’t really matter if that surge was manually or algorithmically driven. Either way the prices were Uber’s doing – and its moral choice.

            Uber has been strenuously defending its surge pricing in the wake of accusations (apparently justified) that the company enjoyed windfall profits during Hurricane Sandy. It has now promised the state of New York that it will cap its surge prices (at three times the highest rate on two non-emergency days). But if Uber has its way, it will soon enjoy a monopolistic stranglehold on car service rates in most major markets. And it has demonstrated its willingness to ignore rules and regulations. That means predictable and affordable taxi fares could become a thing of the past.

            In practice, surge pricing could become a new, privatized form of taxation on middle-class taxi customers.

            Even without surge pricing, Uber and its supporters are hiding its full costs. When middle-class workers are underpaid or deprived of benefits and full working rights, as Uber’s reportedly are, the entire middle-class economy suffers. Overall wages and benefits are suppressed for the majority, while the wealthy few are made even richer. The invisible costs of ventures like Uber are extracted over time, far surpassing whatever short-term savings they may occasionally offer.

            Like Walmart, Uber underpays its employees – many of its drivers are employees, in everything but name – and then drains the social safety net to make up the difference. While Uber preaches libertarianism, it practices a form of corporate welfare. It’s reportedly celebrating Obamacare, for example, since the Affordable Care Act allows it to avoid providing health insurance to its workforce. But the ACA’s subsidies, together with Uber’s often woefully insufficient wages, mean that the rest of us are paying its tab instead. And the lack of income security among Uber’s drivers creates another social cost for Americans – in lost tax revenue, and possibly in increased use of social services.

            The company’s war on regulation will also carry a social price. Uber and its supporters don’t seem to understand that regulations exist for a reason. It’s true that nobody likes excessive bureaucracy, but not all regulations are excessive or onerous. And when they are, it’s a flaw in execution rather than principle.

            Regulations were created because they serve a social purpose, ensuring the free and fair exchange of services and resources among all segments of society. Some services, such as transportation, are of such importance that the public has a vested interest in ensuring they will be readily available at reasonably affordable prices. That’s not unreasonable for taxi services, especially given the fact that they profit from publicly maintained roads and bridges.

            Uber has presented itself as a modernized, efficient alternative to government oversight. But it’s an evasion of regulation, not its replacement. As Alexis Madrigalreports, Uber has deliberately ignored city regulators and used customer demand to force its model of inadequate self-governance (my conclusion, not his) onto one city after another.

            Uber presented itself as a refreshing alternative to the over-bureaucratized world of urban transportation. But that’s a false choice. We can streamline sclerotic city regulators, upgrade taxi fleets and even provide users with fancy apps that make it easier to call a cab. The company’s binary presentation – us, or City Hall – frames the debate in artificial terms.

            Uber claims that its driver rating system is a more efficient way to monitor drivers, but that’s an entirely unproven assumption. While taxi drivers have been known to misbehave, the worldwide litany of complaints against Uber drivers – for everything from dirty cars and spider bites to assault with a hammer, fondling and rape– suggest that Uber’s system may not work as well as old-fashioned regulation. It’s certainly not noticeably superior.

            In fact, prosecutors in San Francisco and Los Angeles say Uber has been lying to its customers about the level and quality of its background checks. The company now promises it will do a better job at screening drivers. But it won’t tell us what measures its taking to improve its safety record, and it’s fighting the kind of driver scrutiny that taxicab companies have been required to enforce for many decades.

            Many reports suggest that beleaguered drivers don’t feel much better about the company than victimized passengers do. They tell horror stories about the company’s hiring and management practices. Uber unilaterally slashes drivers’ rates, while claiming they don’t need to unionize. (The Teamsters disagree.)

            The company also pushes sketchy, substandard loans onto its drivers – but hey, what could go wrong?

            Uber has many libertarian defenders. And yet, it deceives the press and threatens to spy on journalists, lies to its own employees, keeps its practices a secret and routinely invades the privacy of civilians – sometimes merely for entertainment. (It has a tool, with the Orwellian name the “God View,” that it can use for monitoring customers’ personal movements.)

            Aren’t those the kinds of things libertarians say they hate about government?

            This isn’t a “gotcha” exercise. It matters. Uber is the poster child for the pro-privatization, anti-regulatory ideology that ascribes magical powers to technology and the private sector. It is deeply a political entity, from its Nietzschean name to its recent hiring of White House veteran David Plouffe. Uber is built around a relatively simple app (which relies on government-created technology), but it’s not really a tech company. Above all else Uber is an ideological campaign, a neoliberal project whose real products are deregulation and the dismantling of the social contract.

            Or maybe, as that tweeter in Sydney suggested, they’re just assholes.

            Either way, it’s important that Uber’s worldview and business practices not be allowed to “disrupt” our economy or our social fabric. People who work hard deserve to make a decent living. Society at large deserves access to safe and affordable transportation. And government, as the collective expression of a democratic society, has a role to play in protecting its citizens.

            And then there’s the matter of our collective psyche. In her book “A Paradise Built in Hell: The Extraordinary Communities that Arise in Disaster,” Rebecca Solnit wrote of the purpose, meaning and deep satisfaction people find when they pull together to help one another in the face of adversity.  But in the world Uber seeks to create, those surges of the spirit would be replaced by surge pricing.

            You don’t need a “God view” to see what happens next. When heroism is reduced to a transaction, the soul of a society is sold cheap.

            More Richard (R.J.) Eskow.

            You Might Also Like

            More Related Stories

            Featured Slide Shows

            • Share on Twitter
            • Share on Facebook
            • 1 of 5
            • Close
            • Fullscreen
            • Thumbnails

              “One girl can be silenced, but a nation of girls telling their stories becomes free” slideshow

              A photo contest winner

              “One girl can be silenced, but a nation of girls telling their stories becomes free” slideshow

              A photo contest winner

              “One girl can be silenced, but a nation of girls telling their stories becomes free” slideshow

              Superhero Project

              “In life many people have two faces. You think you know someone, but they are not always what they seem. You can’t always trust people. My hero would be someone who is trustworthy, honest and always has their heart in the right place.” Ateya Grade 9 @ Mirman Hayati School (Herat, Afghanistan)

              “One girl can be silenced, but a nation of girls telling their stories becomes free” slideshow

              Superhero Project

              “I pray every night before I go to bed for a hero or an angel capable of helping defenseless children and bringing them happiness. I reach up into the sky hoping to touch a spirit who can make my wish come true.” Fatimah Grade 9 @ Majoba Hervey (Herat, Afghanistan)

            • Recent Slide Shows

            Related Videos

            Comments

            Loading Comments...

            follow salon

            brought to you by

            Most Read

            From Around the Web

            Presented by Zergnet
            ================================================ FILE: test/test-pages/schema-org-context-object/expected-metadata.json ================================================ { "title": "South Korean president’s impeachment fails after ruling party lawmakers walk out", "byline": "Stella Kim, Jennifer Jett", "dir": null, "lang": "en", "excerpt": "South Korean President Yoon Suk Yeol apologized on Saturday for declaring martial law but did not say he would resign as he faces an impeachment vote.", "siteName": "NBC News", "publishedTime": "2024-12-06T22:00:40.000Z", "readerable": true } ================================================ FILE: test/test-pages/schema-org-context-object/expected.html ================================================

            SEOUL, South Korea — A plan to impeach South Korean President Yoon Suk Yeol over his short-lived declaration of martial law has failed after members of his governing party left parliament on Saturday ahead of the planned vote.

            Lawmakers from the People Power Party (PPP) were shouted and cursed at as they left en masse after voting on a separate motion, leaving the 192 opposition lawmakers eight votes short of the two-thirds majority needed for the impeachment motion to succeed.

            Opposition lawmakers had said that if the impeachment motion didn't pass they would come back to it on Wednesday, amid simmering public anger over Yoon’s bewildering order, which plunged the East Asian democracy and key U.S. ally into chaos.

            Earlier Saturday, Yoon apologized for his actions in a brief nationally televised address, saying they “stemmed from the urgency” he felt as president but “caused anxiety and inconvenience to the public.”

            Yoon was still clinging to power on December 5, with his party announcing they will oppose an impeachment motion after his short-lived imposition of martial law stunned the world.
            An image of South Korean President Yoon Suk Yeol is raised during a candlelight rally in Seoul on Thursday.Anthony Wallace / AFP - Getty Images

            “I deeply regret this and sincerely apologize to the citizens who were alarmed,” he said.

            “I will not evade the legal and political responsibility regarding this declaration of martial law,” Yoon continued, speaking for a total of just over two minutes before deeply bowing his head before cameras.

            He said he would not declare martial law a second time, a concern that had been expressed by some opposition lawmakers, and that his party would decide how long he should stay in office and how state affairs would be managed.

            Han Dong-hoon, the leader of the PPP, urged Yoon to step down, telling reporters after the address that it had become “impossible” for him to carry out his normal presidential duties.

            Lee Jae-myung, leader of the liberal opposition Democratic Party and Yoon’s rival in the 2022 presidential election, described the president’s address as “very disappointing.”

            “President Yoon’s statement was completely out of touch with the public’s expectations and further fueled a sense of betrayal,” Lee said.

            “There is no other way but to leave office early.”

            Image: South Korea Descends Into Crisis After Lawmakers Overturn President's Declaration Of Martial Law
            Protesters scuffling with police officers outside the headquarters of Yoon’s People Power Party in Seoul on Friday. Ezra Acayan / Getty Images

            Six opposition parties began impeachment proceedings against Yoon this week. The opposition bloc holds 192 of the unicameral legislature’s 300 seats, just under the two-thirds majority needed for the motion to pass.

            But the conservative PPP also remained formally opposed to a bill appointing a special prosecutor to investigate corruption allegations involving Yoon’s wife. That bill was narrowly voted down on Saturday before PPP lawmakers left the chamber.

            Opposition lawmakers, by contrast, are adamant that Yoon be removed from office, as is much of the public. Large crowds of protesters have called for Yoon’s ouster, while smaller ones have rallied in support of him.

            “You don’t and cannot apologize only with words after attempting to stage martial law. The only way for President Yoon Suk Yeol to mean his apology is to resign now,” Seoul resident Margie Kim said after Yoon’s address.

            A Gallup Korea poll released on Friday showed Yoon’s approval rating at 13%, The Korea Times reported, down from 19% a week earlier.

            Lee, the Democratic Party leader, said Yoon’s “unconstitutional and illegal declaration of martial law” had caused chaos and fear in the country of 50 million people, which spent decades under military-authoritarian rule before transitioning into a vibrant democracy and the world’s 10th-largest economy.

            “The democracy of this nation that has been achieved through blood, sweat and tears is now being wrecked by an unruly power, and our people have suffered serious wounds to their national dignity and pride,” he said Friday.

            Lee also expressed concern about how the martial law declaration might affect South Korea’s relations with the U.S., noting that meetings of the U.S.-South Korea Nuclear Consultative Group that were planned in Washington this week had been postponed amid the turmoil.

            “Our national credibility, as well as diplomacy, has been severely impacted,” he said.

            Biden administration officials said they were not notified in advance that Yoon would declare martial law. But they have repeatedly reaffirmed the “ironclad” nature of the U.S. alliance with South Korea, which it views as an important bulwark against North Korea, China and Russia, and which hosts almost 30,000 American troops.

            Image: South Korea Descends Into Crisis After Lawmakers Overturn President's Declaration Of Martial Law
            South Korea is facing significant political turmoil after Yoon declared martial law, only to lift the order hours later.Ezra Acayan / Getty Images

            The U.S. relationship with South Korea “transcends any particular president or government on both sides of the Pacific Ocean,” State Department deputy spokesperson Vedant Patel said Thursday.

            Before his address on Saturday, Yoon, 63, a former chief prosecutor who took office in 2022, had not made any public appearances since lifting the martial law order early Wednesday after lawmakers defied a security cordon to enter the National Assembly building and voted unanimously to reject it.

            Since then, he has accepted the resignation of his defense minister Kim Yong-hyun, who said he took full responsibility for what happened. Many other officials, including Yoon’s chief of staff and all senior presidential secretaries, have also tendered their resignations.

            South Korean police and prosecutors are investigating claims of insurrection and treason against Yoon, Kim and Interior Minister Lee Sang-min. The Ministry of National Defense has also suspended three top commanders from duty and requested overseas travel bans for them, Kim and seven other active-duty military officers.

            Military officers and government officials described a chaotic six hours from the time Yoon declared martial law around 10:30 p.m. Tuesday to the time he lifted the order around 4:30 a.m. Wednesday. Many of them said they learned about the martial law order, South Korea’s first since 1980, from Yoon’s surprise TV address or from news reports.

            In his address, Yoon accused the opposition-controlled parliament of paralyzing the government by seeking the impeachment of multiple government officials and slashing critical funding from next year’s national budget. He also accused his opponents of sympathizing with nuclear-armed North Korea, its reclusive communist neighbor that the South technically remains at war with.

            A martial law proclamation issued after Yoon’s address banned all political activity, including demonstrations, and censored the news media. It also ordered the country’s striking doctors to return to work within 48 hours.

            Officials said Friday that Yoon had ordered the arrest of prominent politicians including Lee, the leader of the Democratic Party, and Han, the leader of Yoon’s own party, along with other opposition lawmakers, a journalist, a former Supreme Court justice and a former Supreme Court chief justice.

            Yoon’s office has not commented on the allegations.

            Stella Kim reported from Seoul, and Jennifer Jett from Hong Kong.

            Stella Kim

            Stella Kim is an NBC News freelance producer based in Seoul.

            Jennifer Jett

            Jennifer Jett is the Asia Digital Editor for NBC News, based in Hong Kong.

            Abigail Williams and Beomsu Jo contributed.

            ================================================ FILE: test/test-pages/schema-org-context-object/source.html ================================================ South Korean president’s impeachment fails after ruling party lawmakers walk out
            IE 11 is not supported. For an optimal experience visit our site on another browser.

            South Korean president’s impeachment fails after ruling party lawmakers walk out

            President Yoon Suk Yeol had earlier apologized for a short-lived declaration of martial law that plunged the East Asian democracy into chaos, igniting public anger.
            Get more news Live on
            By Stella Kim and Jennifer Jett

            SEOUL, South Korea — A plan to impeach South Korean President Yoon Suk Yeol over his short-lived declaration of martial law has failed after members of his governing party left parliament on Saturday ahead of the planned vote.

            Lawmakers from the People Power Party (PPP) were shouted and cursed at as they left en masse after voting on a separate motion, leaving the 192 opposition lawmakers eight votes short of the two-thirds majority needed for the impeachment motion to succeed.

            Opposition lawmakers had said that if the impeachment motion didn't pass they would come back to it on Wednesday, amid simmering public anger over Yoon’s bewildering order, which plunged the East Asian democracy and key U.S. ally into chaos.

            Earlier Saturday, Yoon apologized for his actions in a brief nationally televised address, saying they “stemmed from the urgency” he felt as president but “caused anxiety and inconvenience to the public.”

            Yoon was still clinging to power on December 5, with his party announcing they will oppose an impeachment motion after his short-lived imposition of martial law stunned the world.
            An image of South Korean President Yoon Suk Yeol is raised during a candlelight rally in Seoul on Thursday.Anthony Wallace / AFP - Getty Images

            “I deeply regret this and sincerely apologize to the citizens who were alarmed,” he said.

            “I will not evade the legal and political responsibility regarding this declaration of martial law,” Yoon continued, speaking for a total of just over two minutes before deeply bowing his head before cameras.

            He said he would not declare martial law a second time, a concern that had been expressed by some opposition lawmakers, and that his party would decide how long he should stay in office and how state affairs would be managed.

            Han Dong-hoon, the leader of the PPP, urged Yoon to step down, telling reporters after the address that it had become “impossible” for him to carry out his normal presidential duties.

            Lee Jae-myung, leader of the liberal opposition Democratic Party and Yoon’s rival in the 2022 presidential election, described the president’s address as “very disappointing.”

            “President Yoon’s statement was completely out of touch with the public’s expectations and further fueled a sense of betrayal,” Lee said.

            “There is no other way but to leave office early.”

            Image: South Korea Descends Into Crisis After Lawmakers Overturn President's Declaration Of Martial Law
            Protesters scuffling with police officers outside the headquarters of Yoon’s People Power Party in Seoul on Friday. Ezra Acayan / Getty Images

            Six opposition parties began impeachment proceedings against Yoon this week. The opposition bloc holds 192 of the unicameral legislature’s 300 seats, just under the two-thirds majority needed for the motion to pass.

            But the conservative PPP also remained formally opposed to a bill appointing a special prosecutor to investigate corruption allegations involving Yoon’s wife. That bill was narrowly voted down on Saturday before PPP lawmakers left the chamber.

            Opposition lawmakers, by contrast, are adamant that Yoon be removed from office, as is much of the public. Large crowds of protesters have called for Yoon’s ouster, while smaller ones have rallied in support of him.

            “You don’t and cannot apologize only with words after attempting to stage martial law. The only way for President Yoon Suk Yeol to mean his apology is to resign now,” Seoul resident Margie Kim said after Yoon’s address.

            A Gallup Korea poll released on Friday showed Yoon’s approval rating at 13%, The Korea Times reported, down from 19% a week earlier.

            Lee, the Democratic Party leader, said Yoon’s “unconstitutional and illegal declaration of martial law” had caused chaos and fear in the country of 50 million people, which spent decades under military-authoritarian rule before transitioning into a vibrant democracy and the world’s 10th-largest economy.

            “The democracy of this nation that has been achieved through blood, sweat and tears is now being wrecked by an unruly power, and our people have suffered serious wounds to their national dignity and pride,” he said Friday.

            Lee also expressed concern about how the martial law declaration might affect South Korea’s relations with the U.S., noting that meetings of the U.S.-South Korea Nuclear Consultative Group that were planned in Washington this week had been postponed amid the turmoil.

            “Our national credibility, as well as diplomacy, has been severely impacted,” he said.

            Biden administration officials said they were not notified in advance that Yoon would declare martial law. But they have repeatedly reaffirmed the “ironclad” nature of the U.S. alliance with South Korea, which it views as an important bulwark against North Korea, China and Russia, and which hosts almost 30,000 American troops.

            Image: South Korea Descends Into Crisis After Lawmakers Overturn President's Declaration Of Martial Law
            South Korea is facing significant political turmoil after Yoon declared martial law, only to lift the order hours later.Ezra Acayan / Getty Images

            The U.S. relationship with South Korea “transcends any particular president or government on both sides of the Pacific Ocean,” State Department deputy spokesperson Vedant Patel said Thursday.

            Before his address on Saturday, Yoon, 63, a former chief prosecutor who took office in 2022, had not made any public appearances since lifting the martial law order early Wednesday after lawmakers defied a security cordon to enter the National Assembly building and voted unanimously to reject it.

            Since then, he has accepted the resignation of his defense minister Kim Yong-hyun, who said he took full responsibility for what happened. Many other officials, including Yoon’s chief of staff and all senior presidential secretaries, have also tendered their resignations.

            South Korean police and prosecutors are investigating claims of insurrection and treason against Yoon, Kim and Interior Minister Lee Sang-min. The Ministry of National Defense has also suspended three top commanders from duty and requested overseas travel bans for them, Kim and seven other active-duty military officers.

            Military officers and government officials described a chaotic six hours from the time Yoon declared martial law around 10:30 p.m. Tuesday to the time he lifted the order around 4:30 a.m. Wednesday. Many of them said they learned about the martial law order, South Korea’s first since 1980, from Yoon’s surprise TV address or from news reports.

            In his address, Yoon accused the opposition-controlled parliament of paralyzing the government by seeking the impeachment of multiple government officials and slashing critical funding from next year’s national budget. He also accused his opponents of sympathizing with nuclear-armed North Korea, its reclusive communist neighbor that the South technically remains at war with.

            A martial law proclamation issued after Yoon’s address banned all political activity, including demonstrations, and censored the news media. It also ordered the country’s striking doctors to return to work within 48 hours.

            Officials said Friday that Yoon had ordered the arrest of prominent politicians including Lee, the leader of the Democratic Party, and Han, the leader of Yoon’s own party, along with other opposition lawmakers, a journalist, a former Supreme Court justice and a former Supreme Court chief justice.

            Yoon’s office has not commented on the allegations.

            Stella Kim reported from Seoul, and Jennifer Jett from Hong Kong.

            Stella Kim

            Stella Kim is an NBC News freelance producer based in Seoul.

            Jennifer Jett

            Jennifer Jett is the Asia Digital Editor for NBC News, based in Hong Kong.

            Abigail Williams and Beomsu Jo contributed.
            ================================================ FILE: test/test-pages/seattletimes-1/expected-metadata.json ================================================ { "title": "Alaskan halibut, caught by a century-old Seattle boat, provides a glimpse of Amazon’s strategy with Whole Foods", "byline": "April 28, 2019 at 6:01 am Updated April 29, 2019 at 3:33 pm", "dir": null, "lang": "en-US", "excerpt": "The story of Whole Foods’ halibut deal opens a window into Amazon’s grocery strategy and draws a line from a Seattle industry with roots in the 19th century to the dominant economic force of the 21st.", "siteName": "The Seattle Times", "publishedTime": "2019-04-28 06:01:07", "readerable": true } ================================================ FILE: test/test-pages/seattletimes-1/expected.html ================================================

            From the deck of his 106-year-old halibut schooner, undergoing a seasonal overhaul at Fisherman’s Terminal in Seattle, skipper Wade Bassi has better insight than most into what’s happening at Amazon-owned Whole Foods Market, at least as pertains to the product he knows best.

            While he doesn’t buy halibut much — he’s got a freezer full of it — Bassi, 43 years a fisherman, keeps an eye on how it’s handled and presented in the grocery stores and fish markets.

            “When you look at nice halibut, I mean it is pure white,” he said. “And it is flaky-looking, and it is beautiful. It’s translucent. If you’ve got that in the fish market, people are going to buy it.”

            A few days earlier, Whole Foods touted a rarely seen promotional price for halibut as part of its ongoing campaign to revise the grocery chain’s high-cost reputation while maintaining its image for quality and sustainability.

            “Whole Foods is one of the better ones, to be honest with you,” Bassi said. “But you know, Whole Foods, whole paycheck. … They usually do charge way more for everything than anywhere else. Which really surprises me that they’re selling this for $16-something a pound, because they’re not making anything on it.”

            Whole Foods’ halibut deal opens a window into Amazon’s grocery strategy as it seeks to combine the defining characteristics of each brand, leverage its juggernaut Prime membership program and take a larger share of the grocery business from competitors such as Walmart, Kroger and Costco.

            It also draws a long line from a major Seattle industry with roots in the 19th century to the dominant economic force of the 21st.

            Amazon bought Whole Foods in August 2017 for $13.7 billion, its largest acquisition and an aggressive move into the grocery business.

            The combination of the two has been steady, said Tom Forte, who follows Amazon as a managing director at the D.A. Davidson brokerage. In a few more years, he said, “You won’t recognize the original Whole Foods.”

            Within months of the acquisition, Forte said, Whole Foods was selling cheaper cage-free eggs and organic ground beef, prices it said were a result of the deal.

            Then came the integration of Prime, Amazon’s $119-a-year shipping and media-subscription program, which Amazon founder Jeff Bezos said last year had surpassed 100 million members. In Whole Foods, Prime operates as a hybrid of the customer-loyalty discount programs offered by most grocers — in which consumers trade details of their purchasing habits for lower prices — and a paid membership like at Costco or Sam’s Club.

            Whole Foods stores have been festooned with yellow and blue signs pointing out Prime member benefits, one of which was fresh halibut fillets priced at $16.99 a pound, albeit only for a week earlier this month.

            “I was shocked to see that level,” said Tyler Besecker, president of Mercer Island-based Dana F. Besecker Company, the largest buyer of Pacific halibut. The price, which was matched at Kroger-owned QFC stores in the region last week, is “as low as I’ve ever seen.” (Besecker does not currently supply Whole Foods.)

            Fresh halibut fillets routinely sell for $24 to $28 a pound, and often more.

            He said there’s little if any room for a profit at the promotional price offered by Whole Foods and QFC. “They might be selling those at cost or as loss leaders just to get people into the stores,” Besecker said.

            In the competitive grocery business, promotions like this happen all the time. The thinking is that shoppers will be attracted by the discount on a staple or a prestige item, and then fill their carts with other groceries sold at a profit.

            A Whole Foods spokeswoman declined to comment on pricing. The temporary halibut discount is one of more than 300 such Prime promotions Whole Foods plans in the next few months. The company also said it was lowering prices across the store, its third such announcement since the acquisition.

            At the seafood counter in the Whole Foods store on Westlake, surrounded by Amazon headquarters buildings, a sign advertised “First of the Season Fresh Alaskan Halibut” and sported the blue Marine Stewardship Council (MSC) Certified Sustainable Seafood label.

            Whole Foods has been a pioneer in sustainable-seafood marketing, beginning in 1999 when it began to stock fish with the MSC label. In the mid-2000s, Pacific halibut fishermen sought the certification — a system of third-party audits that tracks seafood from catch to market — and Whole Foods was there from the beginning.

            “They were the first ones to market the MSC halibut,” said Bob Alverson, head of the Fishing Vessel Owners Association, representing boats that catch halibut and black cod and a driver of the certification effort. “It turned into quite a marketing advantage. Whole Foods saw that early. They were focusing on sustainable, high-quality food products. They had quite a bit of foresight, I think, in that direction.”

            The certification comes with added costs borne by the fishermen and buyers, and passed on to consumers. But it’s also an assurance “that people are watching out for the resource,” he said.

            As it tries to convince people it has lower prices, Whole Foods has been very careful to maintain the reputation built on products like MSC-certified halibut.

            Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)
            Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)

            Whole Foods future

            New signs in stores appeared this month, spelling out the value proposition it’s trying to strike: “New lower prices. Same high standards,” reads one, against a background image of carrots.

            At the same time, the company claims its new prices and Prime deals have saved customers “hundreds of millions of dollars” since the Amazon acquisition.

            If prices are being lowered and the quality bar stays the same, something else has to give.

            Analysts provided a few theories:

            Amazon could be willing to accept losses or slimmer profits within Whole Foods, as it has done in other businesses, in an effort to expand its customer base.

            “Do they take the profit from their non-retail efforts, which today is primarily cloud computing, and then reinvest those profits to take share in grocery?” Forte said.

            That could eventually open up an avenue to growth as the rate of expansion slows in Amazon’s broader U.S. retail sales.

            “That’s why they need grocery to work,” Forte said. “Grocery’s a very big category.”

            Whole Foods could also press suppliers to reduce their prices, essentially cutting their profit.

            “The worry with that is that it puts the squeeze on the producer upstream,” said Ananth Iyer, a professor at Purdue University whose research includes sustainability in supply chains. If producers are squeezed too much, he noted, they may start to cut corners.

            So far, there’s no evidence this is happening in halibut, where fishery practices have been carefully managed with a goal of sustainability for nearly a century. Also, Whole Foods does not yet have the scale as a buyer to dictate prices the way a company like Costco does.

            Forte said that even if it did have such clout, this would be a risky strategy that would undermine the very attributes of the Whole Foods brand that make it most valuable.

            Another theory is that Amazon could apply more of its technology and expertise in logistics to create supply-chain efficiencies that would maintain its profits while benefiting producers and consumers, particularly with perishable grocery products, Iyer said. This is part of the promise of the acquisition in the first place.

            “That’s a powerful combination,” Forte said. “The sustainable, the organic, the healthier food — all those qualities of Whole Foods, with the supply-chain technology of Amazon. It plays to the strengths of both sides.”

            Forte said he expects Amazon to continue its aggressive moves on grocery pricing at Whole Foods. But he wonders when the price cuts will be broader, particularly as Amazon competes with the likes of Walmart and Kroger for a bigger slice of U.S. food and beverage retail sales, which totaled $726 billion in 2017. Whole Foods said it has lowered prices on hundreds of items, with an emphasis on fresh produce.

            Forte described his attempt after the acquisition to get the ingredients for Rice Krispies Treats at Whole Foods. It was perhaps doomed from the start: Whole Foods doesn’t carry Rice Krispies. But he found an organic brown rice puff cereal and organic marshmallows. They were “so wildly expensive that we didn’t finish the exercise. I took the kids to Walmart and bought the ingredients for a pittance,” Forte said.

            That points to the bigger question of how Amazon plans to position Whole Foods for the long term in its expanding array of physical retail-grocery formats. It now has 11 automated Go convenience stores and is rumored to be planning a new, low-priced grocery chain of its own that may deploy the same cashierless checkout technology.

            Meanwhile, Whole Foods is not opening any new 365 stores, a smaller, lower-priced version of the main brand highlighting the company’s private-label products. Whole Foods co-founder and CEO John Mackey said in an internal memo that the “price distinction between the two brands has become less relevant” as Whole Foods lowered its prices, Yahoo Finance reported earlier this year.

            Amazon, too, is finding success with a growing stable of private-label brands — it had more than 100 as of last July, according to Coresight Research, double the number in 2017. One of these, Solimo, sells generic versions of everything from K-Cup coffee pods to Epsom salts to garbage bags and racked up more than $6 million in sales in January alone, according to data analysis firm 1010data.

            Of course, there’s nothing generic about a “fresh, sustainable wild-caught halibut fillet.”

            Back on the schooner

            The day after Easter, Bassi and his crew — three family members and an unrelated father-and-son team — loaded up the Polaris, one of four century-old wooden schooners still chasing halibut out of Seattle. (The broader Washington-based halibut fleet numbers about 100 vessels.) Bassi’s father fished on the Polaris, which Bassi co-owns with Rolfe McCartney. Bassi’s grandfather fished halibut back when schooners carried small dories out to the fishing grounds, which made the landing of a fish that can grow to 500 pounds all the more exciting.

            The Polaris motored out of Fisherman’s Terminal and through the Ballard Locks to begin the three-day journey through the Inside Passage to Ketchikan, Alaska. There, they take on tons of ice and bait, herring for the black cod Bassi will target first, and later chum salmon, codfish or octopus for the halibut.

            From a base in Kodiak, Alaska, the Polaris makes a series of trips, at sea for a week or longer at a time, to fish as far away as Attu Island at the far western edge of the Aleutian Islands chain. “It’s a big range that we fish,” Bassi said.

            The Polaris will trail long lines of hooks, leaving them to soak for several hours before reeling them in. The fish are stunned, bled and dressed, and put on ice in the hold. It is this fishing method that contributes to the quality of the halibut and sustainability of the fishery, as it reduces by-catch — the inadvertent taking of other species.

            They negotiate to sell the fish with four or five buyers, such as Besecker, at a price that fluctuates throughout the season. This is a peak time of year for halibut, with consumers seeking out fresh fish for Easter and Mother’s Day, Besecker said. Fishermen are typically paid between $5 and $6 a pound for halibut.

            The Alaskan halibut fishery has its troubles — as nearly all fisheries do — but has been rationalized and managed successfully, particularly over the last quarter-century. Alverson, a commissioner on the Seattle-based International Pacific Halibut Commission, which has managed catch limits for U.S. and Canadian fishermen since 1924, described it as a stable but declining resource.

            This year, the halibut fishery in the Northern Pacific and Bering Sea is capped at 29.4 million pounds, with most of that allocated to commercial fishing and smaller amounts reserved for recreational and tribal fishing, as well as by-catch of other commercial fisheries.

            Alverson summed up the journey from a wooden fishing boat in the Bering Sea to the fish counter of a grocery store owned by a company that has redefined modern buying and selling: “It’s Seattle old school meets Seattle new school with Amazon.”

            ================================================ FILE: test/test-pages/seattletimes-1/source.html ================================================ Alaskan halibut, caught by a century-old Seattle boat, provides a glimpse of Amazon’s strategy with Whole Foods | The Seattle Times
            Amazon
            Amazon

            Alaskan halibut, caught by a century-old Seattle boat, provides a glimpse of Amazon’s strategy with Whole Foods

            Benjamin Romano
            By
            Seattle Times business reporter

            From the deck of his 106-year-old halibut schooner, undergoing a seasonal overhaul at Fisherman’s Terminal in Seattle, skipper Wade Bassi has better insight than most into what’s happening at Amazon-owned Whole Foods Market, at least as pertains to the product he knows best.

            While he doesn’t buy halibut much — he’s got a freezer full of it — Bassi, 43 years a fisherman, keeps an eye on how it’s handled and presented in the grocery stores and fish markets.

            “When you look at nice halibut, I mean it is pure white,” he said. “And it is flaky-looking, and it is beautiful. It’s translucent. If you’ve got that in the fish market, people are going to buy it.”

            A few days earlier, Whole Foods touted a rarely seen promotional price for halibut as part of its ongoing campaign to revise the grocery chain’s high-cost reputation while maintaining its image for quality and sustainability.

            “Whole Foods is one of the better ones, to be honest with you,” Bassi said. “But you know, Whole Foods, whole paycheck. … They usually do charge way more for everything than anywhere else. Which really surprises me that they’re selling this for $16-something a pound, because they’re not making anything on it.”

            Whole Foods’ halibut deal opens a window into Amazon’s grocery strategy as it seeks to combine the defining characteristics of each brand, leverage its juggernaut Prime membership program and take a larger share of the grocery business from competitors such as Walmart, Kroger and Costco.

            Advertising

            It also draws a long line from a major Seattle industry with roots in the 19th century to the dominant economic force of the 21st.

            Amazon bought Whole Foods in August 2017 for $13.7 billion, its largest acquisition and an aggressive move into the grocery business.

            The combination of the two has been steady, said Tom Forte, who follows Amazon as a managing director at the D.A. Davidson brokerage. In a few more years, he said, “You won’t recognize the original Whole Foods.”

            Within months of the acquisition, Forte said, Whole Foods was selling cheaper cage-free eggs and organic ground beef, prices it said were a result of the deal.

            Then came the integration of Prime, Amazon’s $119-a-year shipping and media-subscription program, which Amazon founder Jeff Bezos said last year had surpassed 100 million members. In Whole Foods, Prime operates as a hybrid of the customer-loyalty discount programs offered by most grocers — in which consumers trade details of their purchasing habits for lower prices — and a paid membership like at Costco or Sam’s Club.

            Whole Foods stores have been festooned with yellow and blue signs pointing out Prime member benefits, one of which was fresh halibut fillets priced at $16.99 a pound, albeit only for a week earlier this month.

            Advertising

            “I was shocked to see that level,” said Tyler Besecker, president of Mercer Island-based Dana F. Besecker Company, the largest buyer of Pacific halibut. The price, which was matched at Kroger-owned QFC stores in the region last week, is “as low as I’ve ever seen.” (Besecker does not currently supply Whole Foods.)

            Fresh halibut fillets routinely sell for $24 to $28 a pound, and often more.

            He said there’s little if any room for a profit at the promotional price offered by Whole Foods and QFC. “They might be selling those at cost or as loss leaders just to get people into the stores,” Besecker said.

            In the competitive grocery business, promotions like this happen all the time. The thinking is that shoppers will be attracted by the discount on a staple or a prestige item, and then fill their carts with other groceries sold at a profit.

            A Whole Foods spokeswoman declined to comment on pricing. The temporary halibut discount is one of more than 300 such Prime promotions Whole Foods plans in the next few months. The company also said it was lowering prices across the store, its third such announcement since the acquisition.

            At the seafood counter in the Whole Foods store on Westlake, surrounded by Amazon headquarters buildings, a sign advertised “First of the Season Fresh Alaskan Halibut” and sported the blue Marine Stewardship Council (MSC) Certified Sustainable Seafood label.

            Whole Foods has been a pioneer in sustainable-seafood marketing, beginning in 1999 when it began to stock fish with the MSC label. In the mid-2000s, Pacific halibut fishermen sought the certification — a system of third-party audits that tracks seafood from catch to market — and Whole Foods was there from the beginning.

            “They were the first ones to market the MSC halibut,” said Bob Alverson, head of the Fishing Vessel Owners Association, representing boats that catch halibut and black cod and a driver of the certification effort. “It turned into quite a marketing advantage. Whole Foods saw that early. They were focusing on sustainable, high-quality food products. They had quite a bit of foresight, I think, in that direction.”

            The certification comes with added costs borne by the fishermen and buyers, and passed on to consumers. But it’s also an assurance “that people are watching out for the resource,” he said.

            Most Read Business Stories

            As it tries to convince people it has lower prices, Whole Foods has been very careful to maintain the reputation built on products like MSC-certified halibut.

            Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)
            Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)

            Whole Foods future

            New signs in stores appeared this month, spelling out the value proposition it’s trying to strike: “New lower prices. Same high standards,” reads one, against a background image of carrots.

            At the same time, the company claims its new prices and Prime deals have saved customers “hundreds of millions of dollars” since the Amazon acquisition.

            Advertising

            If prices are being lowered and the quality bar stays the same, something else has to give.

            Analysts provided a few theories:

            Amazon could be willing to accept losses or slimmer profits within Whole Foods, as it has done in other businesses, in an effort to expand its customer base.

            “Do they take the profit from their non-retail efforts, which today is primarily cloud computing, and then reinvest those profits to take share in grocery?” Forte said.

            That could eventually open up an avenue to growth as the rate of expansion slows in Amazon’s broader U.S. retail sales.

            “That’s why they need grocery to work,” Forte said. “Grocery’s a very big category.”

            Whole Foods could also press suppliers to reduce their prices, essentially cutting their profit.

            Advertising

            “The worry with that is that it puts the squeeze on the producer upstream,” said Ananth Iyer, a professor at Purdue University whose research includes sustainability in supply chains. If producers are squeezed too much, he noted, they may start to cut corners.

            So far, there’s no evidence this is happening in halibut, where fishery practices have been carefully managed with a goal of sustainability for nearly a century. Also, Whole Foods does not yet have the scale as a buyer to dictate prices the way a company like Costco does.

            Forte said that even if it did have such clout, this would be a risky strategy that would undermine the very attributes of the Whole Foods brand that make it most valuable.

            Another theory is that Amazon could apply more of its technology and expertise in logistics to create supply-chain efficiencies that would maintain its profits while benefiting producers and consumers, particularly with perishable grocery products, Iyer said. This is part of the promise of the acquisition in the first place.

            “That’s a powerful combination,” Forte said. “The sustainable, the organic, the healthier food — all those qualities of Whole Foods, with the supply-chain technology of Amazon. It plays to the strengths of both sides.”

            Forte said he expects Amazon to continue its aggressive moves on grocery pricing at Whole Foods. But he wonders when the price cuts will be broader, particularly as Amazon competes with the likes of Walmart and Kroger for a bigger slice of U.S. food and beverage retail sales, which totaled $726 billion in 2017. Whole Foods said it has lowered prices on hundreds of items, with an emphasis on fresh produce.

            Advertising

            Forte described his attempt after the acquisition to get the ingredients for Rice Krispies Treats at Whole Foods. It was perhaps doomed from the start: Whole Foods doesn’t carry Rice Krispies. But he found an organic brown rice puff cereal and organic marshmallows. They were “so wildly expensive that we didn’t finish the exercise. I took the kids to Walmart and bought the ingredients for a pittance,” Forte said.

            That points to the bigger question of how Amazon plans to position Whole Foods for the long term in its expanding array of physical retail-grocery formats. It now has 11 automated Go convenience stores and is rumored to be planning a new, low-priced grocery chain of its own that may deploy the same cashierless checkout technology.

            Meanwhile, Whole Foods is not opening any new 365 stores, a smaller, lower-priced version of the main brand highlighting the company’s private-label products. Whole Foods co-founder and CEO John Mackey said in an internal memo that the “price distinction between the two brands has become less relevant” as Whole Foods lowered its prices, Yahoo Finance reported earlier this year.

            Amazon, too, is finding success with a growing stable of private-label brands — it had more than 100 as of last July, according to Coresight Research, double the number in 2017. One of these, Solimo, sells generic versions of everything from K-Cup coffee pods to Epsom salts to garbage bags and racked up more than $6 million in sales in January alone, according to data analysis firm 1010data.

            Of course, there’s nothing generic about a “fresh, sustainable wild-caught halibut fillet.”

            Back on the schooner

            The day after Easter, Bassi and his crew — three family members and an unrelated father-and-son team — loaded up the Polaris, one of four century-old wooden schooners still chasing halibut out of Seattle. (The broader Washington-based halibut fleet numbers about 100 vessels.) Bassi’s father fished on the Polaris, which Bassi co-owns with Rolfe McCartney. Bassi’s grandfather fished halibut back when schooners carried small dories out to the fishing grounds, which made the landing of a fish that can grow to 500 pounds all the more exciting.

            Advertising

            The Polaris motored out of Fisherman’s Terminal and through the Ballard Locks to begin the three-day journey through the Inside Passage to Ketchikan, Alaska. There, they take on tons of ice and bait, herring for the black cod Bassi will target first, and later chum salmon, codfish or octopus for the halibut.

            From a base in Kodiak, Alaska, the Polaris makes a series of trips, at sea for a week or longer at a time, to fish as far away as Attu Island at the far western edge of the Aleutian Islands chain. “It’s a big range that we fish,” Bassi said.

            The Polaris will trail long lines of hooks, leaving them to soak for several hours before reeling them in. The fish are stunned, bled and dressed, and put on ice in the hold. It is this fishing method that contributes to the quality of the halibut and sustainability of the fishery, as it reduces by-catch — the inadvertent taking of other species.

            They negotiate to sell the fish with four or five buyers, such as Besecker, at a price that fluctuates throughout the season. This is a peak time of year for halibut, with consumers seeking out fresh fish for Easter and Mother’s Day, Besecker said. Fishermen are typically paid between $5 and $6 a pound for halibut.

            The Alaskan halibut fishery has its troubles — as nearly all fisheries do — but has been rationalized and managed successfully, particularly over the last quarter-century. Alverson, a commissioner on the Seattle-based International Pacific Halibut Commission, which has managed catch limits for U.S. and Canadian fishermen since 1924, described it as a stable but declining resource.

            This year, the halibut fishery in the Northern Pacific and Bering Sea is capped at 29.4 million pounds, with most of that allocated to commercial fishing and smaller amounts reserved for recreational and tribal fishing, as well as by-catch of other commercial fisheries.

            Advertising

            Alverson summed up the journey from a wooden fishing boat in the Bering Sea to the fish counter of a grocery store owned by a company that has redefined modern buying and selling: “It’s Seattle old school meets Seattle new school with Amazon.”

            Related

            More on Amazon

              More

              Benjamin Romano: 206-652-6593 or bromano@seattletimes.com; on Twitter: @bromano.
              View Comments
              You must be logged in to leave a comment. or .

              The opinions expressed in reader comments are those of the author only, and do not reflect the opinions of The Seattle Times.

              Advertising
              ================================================ FILE: test/test-pages/simplyfound-1/expected-metadata.json ================================================ { "title": "Raspberry Pi 3 - The credit card sized PC that cost only $35 - All-time bestselling computer in UK", "byline": null, "dir": null, "lang": "en", "excerpt": "The Raspberry Pi Foundation started by a handful of volunteers in 2012 when they released the original Raspberry Pi 256MB Model B without knowing what to expect. In a short four-year period they have grown to over sixty full-time employees and ha...", "siteName": "SIMPLYFOUND.COM | BY: JOE WEE", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/simplyfound-1/expected.html ================================================

              The Raspberry Pi Foundation started by a handful of volunteers in 2012 when they released the original Raspberry Pi 256MB Model B without knowing what to expect.  In a short four-year period they have grown to over sixty full-time employees and have shipped over eight million units to-date.  Raspberry Pi has achieved new heights by being shipped to the International Space Station for research and by being an affordable computing platforms used by teachers throughout the world.  "It has become the all-time best-selling computer in the UK".

              Raspberry Pi 3 - A credit card sized PC that only costs $35 - Image: Raspberry Pi Foundation

              Raspberry Pi Foundation is charity organization that pushes for a digital revolution with a mission to inspire kids to learn by creating computer-powered objects.  The foundation also helps teachers learn computing  skills through free training and readily available tutorials & example code for creating cool things such as music.

              Raspberry Pi in educations - Image: Raspberry Pi Foundation

              In celebration of their 4th year anniversary, the foundation has released Raspberry Pi 3 with the same price tag of $35 USD.  The 3rd revision features a 1.2GHz 64-bit quad-core ARM CPU with integrated Bluetooth 4.1 and 802.11n wireless LAN chipsets.  The ARM Cortex-A53 CPU along with other architectural enhancements making it the fastest Raspberry Pi to-date.  The 3rd revision is reportedly about 50-60% times faster than its predecessor Raspberry Pi 2 and about 10 times faster then the original Raspberry PI.

              Raspberry Pi - Various Usage

              Raspberry Pi 3 is now available via many online resellers.  At this time, you should use a recent 32-bit NOOBS or Raspbian image from their downloads page with a promise of a switch to a 64-bit version only if further investigation proves that there is indeed some value in moving to 64-bit mode.

              ================================================ FILE: test/test-pages/simplyfound-1/source.html ================================================ Raspberry Pi 3 - The credit card sized PC that cost only $35 - All-time bestselling computer in UK - SimplyFound

              Raspberry Pi 3 - The credit card sized PC that cost only $35 - All-time bestselling computer in UK

              Joe Wee     Monday, February 29, 2016 @ 11:10 PM UTC
              Consumer Electronics Technology PC Raspberry Pi Computer Raspberry Pi 3

              The Raspberry Pi Foundation started by a handful of volunteers in 2012 when they released the original Raspberry Pi 256MB Model B without knowing what to expect.  In a short four-year period they have grown to over sixty full-time employees and have shipped over eight million units to-date.  Raspberry Pi has achieved new heights by being shipped to the International Space Station for research and by being an affordable computing platforms used by teachers throughout the world.  "It has become the all-time best-selling computer in the UK".

              Raspberry Pi 3 - A credit card sized PC that only costs $35 - Image: Raspberry Pi Foundation

              Raspberry Pi Foundation is charity organization that pushes for a digital revolution with a mission to inspire kids to learn by creating computer-powered objects.  The foundation also helps teachers learn computing  skills through free training and readily available tutorials & example code for creating cool things such as music.

              Raspberry Pi in educations - Image: Raspberry Pi Foundation

              In celebration of their 4th year anniversary, the foundation has released Raspberry Pi 3 with the same price tag of $35 USD.  The 3rd revision features a 1.2GHz 64-bit quad-core ARM CPU with integrated Bluetooth 4.1 and 802.11n wireless LAN chipsets.  The ARM Cortex-A53 CPU along with other architectural enhancements making it the fastest Raspberry Pi to-date.  The 3rd revision is reportedly about 50-60% times faster than its predecessor Raspberry Pi 2 and about 10 times faster then the original Raspberry PI.

              Raspberry Pi - Various Usage

              Raspberry Pi 3 is now available via many online resellers.  At this time, you should use a recent 32-bit NOOBS or Raspbian image from their downloads page with a promise of a switch to a 64-bit version only if further investigation proves that there is indeed some value in moving to 64-bit mode.

              137
              1 Share
              © 2016   Simplyfound.com
              ================================================ FILE: test/test-pages/social-buttons/expected-metadata.json ================================================ { "title": "Share buttons removal test", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/social-buttons/expected.html ================================================

              Lorem ipsum dolor

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/social-buttons/source.html ================================================ Share buttons removal test

              Lorem ipsum dolor

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/spiceworks/expected-metadata.json ================================================ { "title": "Rewriting Rules of Engagement with Video in 2020: Vidyard Introduces New Features on its Video Platform", "byline": "Last Updated: January 7, 2025", "dir": null, "lang": "en-US", "excerpt": "Vidyard launches new tools and features on its platform to help professionals collaborate and share videos on the go.", "siteName": "Spiceworks", "publishedTime": "2020-07-10T14:15:42+00:00", "readerable": true } ================================================ FILE: test/test-pages/spiceworks/expected.html ================================================

              Vidyard Launches New Tools and Features for Businesses

              Vidyard announced significant updates to its video hosting platform and tools yesterday. The company that provides video platforms for businesses saidOpens a new window that the new product lineup was introduced in response to brands’ rapidly increasing reliance on videos to reach and engage remote audiences. In light of COVID-19 and resulting changes in the way we work, live, and buy – videos are no longer just a marketing tool, they are critical for employee engagement, prospecting, and sales.

              Key changes to Vidyard’s platform include new integration with Zoom, redesign of its video hosting platform, and enterprise video platform services. It is interesting to see that the company is offering business-grade video hosting, creation, sharing, and measurement capabilities for free to all users.

              Learn More: Video Marketing 101: 4 Basics Marketers Can’t Miss

              “Businesses are quickly evolving their sales, marketing, and communications strategies to align with a digital-first communication world,” says Michael Litt, co-founder and CEO of Vidyard. “Video is a huge part of that as it’s simply more efficient, more expressive, and more effective than standard text. Today’s businesses need a simpler, smarter, and more scalable way to put the power of video into their people’s hands. We’re excited to deliver on that need with a new breed of video creation and hosting solutions that turn any business professional into a video creator and any organization into a video-first business.”

              The company witnessed a 400&percnt; increase in new monthly sign-ups of its online screen recording and video creation tool since March 1, 2020. With this release, Vidyard has launched some innovative new features on its platform such as online video recording and sharing that allow users to create and collaborate on video creation. It has also introduced new mobile apps to enable marketers to create and share customer videos from anywhere or on the go. New launches include:

              • Vidyard android app and Updated iOS app: These apps have been designed to help marketers and professionals record and share videos from anywhere. The apps allow users to quickly record videos while on the go and immediately share them through a company-branded video page. The mobile apps and the online video recording tool connect to the same video libraries, making it seamless for users to transition between desktop and mobile with continuous access to all video recordings.
              • Comments and reply features: Viewers can now leave a comment or reply to videos shared by professionals using Pro, Teams, Business, and Enterprise tier offerings. Video posters/sharers receive comment notifications to enable quicker replies – thereby turning any video into a collaboration tool.
              • Improved security features: Vidyard now offers users of Pro, Teams, and Business plans to secure viewer access to private videos through custom passwords. Customers can also use single sign-on (SSO) to ensure private videos created and shared by internal users can be viewed by other employees within the organization.
              • Revamped UX: Vidyard makes it easier for users to generate transcripts, add closed captions, customize video thumbnail, set viewing permissions, and publish content across public and private channels. The company says it has made it easier for beginner video creators to create professional-grade videos easily.

              Learn More: Seven Tips for Repurposing Video Content

              Video is the Ultimate Engagement Tool for Marketers in 2020

              With people confined to their homes for the last three to four months, our social lives have moved online, and video consumption has risen notably within the at-home segments of television, gaming, education, OTT, and business content.

              Consumer behavior is rapidly evolving as the world adjusts to the new normal. The emergence of trends, such as social distancing, work-from-home, and virtual meetings have increased the demand for at-home digital media.

              Video platforms like Vidyard are looking to accelerate consumer habit-formation through the ease of access and intuitive interfaces. What does this mean for marketers? It is no secret that videos are the most engaging tool available to marketers today. However, barriers such as video production, screen recording, and easy sharing had traditionally prevented marketers from unlocking the real potential of digital videos.

              With the introduction of smart video platforms, marketers can leverage the format to not only engage customers and drive sales, but also share access with other business functions within the organization, like HR and finance, who can also use it as an employee-facing tool.

              ================================================ FILE: test/test-pages/spiceworks/source.html ================================================ Rewriting Rules of Engagement with Video in 2020: Vidyard Introduces New Features on its Video Platform - Spiceworks
              Home
              Marketing Strategy

              Rewriting Rules of Engagement with Video in 2020: Vidyard Introduces New Features on its Video Platform

              Last Updated: January 7, 2025


              Vidyard Launches New Tools and Features for Businesses

              Vidyard announced significant updates to its video hosting platform and tools yesterday. The company that provides video platforms for businesses saidOpens a new window that the new product lineup was introduced in response to brands’ rapidly increasing reliance on videos to reach and engage remote audiences. In light of COVID-19 and resulting changes in the way we work, live, and buy – videos are no longer just a marketing tool, they are critical for employee engagement, prospecting, and sales.

              Key changes to Vidyard’s platform include new integration with Zoom, redesign of its video hosting platform, and enterprise video platform services. It is interesting to see that the company is offering business-grade video hosting, creation, sharing, and measurement capabilities for free to all users.

              Learn More: Video Marketing 101: 4 Basics Marketers Can’t Miss

              “Businesses are quickly evolving their sales, marketing, and communications strategies to align with a digital-first communication world,” says Michael Litt, co-founder and CEO of Vidyard. “Video is a huge part of that as it’s simply more efficient, more expressive, and more effective than standard text. Today’s businesses need a simpler, smarter, and more scalable way to put the power of video into their people’s hands. We’re excited to deliver on that need with a new breed of video creation and hosting solutions that turn any business professional into a video creator and any organization into a video-first business.”

              The company witnessed a 400&percnt; increase in new monthly sign-ups of its online screen recording and video creation tool since March 1, 2020. With this release, Vidyard has launched some innovative new features on its platform such as online video recording and sharing that allow users to create and collaborate on video creation. It has also introduced new mobile apps to enable marketers to create and share customer videos from anywhere or on the go. New launches include:

              • Vidyard android app and Updated iOS app: These apps have been designed to help marketers and professionals record and share videos from anywhere. The apps allow users to quickly record videos while on the go and immediately share them through a company-branded video page. The mobile apps and the online video recording tool connect to the same video libraries, making it seamless for users to transition between desktop and mobile with continuous access to all video recordings.
              • Comments and reply features: Viewers can now leave a comment or reply to videos shared by professionals using Pro, Teams, Business, and Enterprise tier offerings. Video posters/sharers receive comment notifications to enable quicker replies – thereby turning any video into a collaboration tool.
              • Improved security features: Vidyard now offers users of Pro, Teams, and Business plans to secure viewer access to private videos through custom passwords. Customers can also use single sign-on (SSO) to ensure private videos created and shared by internal users can be viewed by other employees within the organization.
              • Revamped UX: Vidyard makes it easier for users to generate transcripts, add closed captions, customize video thumbnail, set viewing permissions, and publish content across public and private channels. The company says it has made it easier for beginner video creators to create professional-grade videos easily.

              Learn More: Seven Tips for Repurposing Video Content

              Video is the Ultimate Engagement Tool for Marketers in 2020

              With people confined to their homes for the last three to four months, our social lives have moved online, and video consumption has risen notably within the at-home segments of television, gaming, education, OTT, and business content.

              Consumer behavior is rapidly evolving as the world adjusts to the new normal. The emergence of trends, such as social distancing, work-from-home, and virtual meetings have increased the demand for at-home digital media.

              Video platforms like Vidyard are looking to accelerate consumer habit-formation through the ease of access and intuitive interfaces. What does this mean for marketers? It is no secret that videos are the most engaging tool available to marketers today. However, barriers such as video production, screen recording, and easy sharing had traditionally prevented marketers from unlocking the real potential of digital videos.

              With the introduction of smart video platforms, marketers can leverage the format to not only engage customers and drive sales, but also share access with other business functions within the organization, like HR and finance, who can also use it as an employee-facing tool.

              Take me to Community
              Do you still have questions? Head over to the Spiceworks Community to find answers.

              Recommended Reads

              Stasis Is Not An Option: Now’s The Time To Adapt And Find The New Models We Need To Thrive
              Marketing Strategy

              Stasis Is Not An Option: Now’s The Time To Adapt And Find The New Models We Need To Thrive

              Revolutionizing Healthcare with AI: Key Strategies for Medtech
              Artificial Intelligence

              Revolutionizing Healthcare with AI: Key Strategies for Medtech

              AI Ascendant: Key Takeaways from Google I/O 2024 for Modern Marketers
              AI in Marketing

              AI Ascendant: Key Takeaways from Google I/O 2024 for Modern Marketers

              How to Move Beyond Third-Party Tracking in 2024
              Customer Data Management

              How to Move Beyond Third-Party Tracking in 2024

              Precision Marketing in Healthcare With Real-time Clinical Data
              Customer Data Management

              Precision Marketing in Healthcare With Real-time Clinical Data

              Meta to Sunset Workplace by June 2026
              Marketing Strategy

              Meta to Sunset Workplace by June 2026

              ================================================ FILE: test/test-pages/style-tags-removal/expected-metadata.json ================================================ { "title": "Style tags removal", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/style-tags-removal/expected.html ================================================

              Lorem

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Foo

              Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/style-tags-removal/source.html ================================================ Style tags removal

              Lorem

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Foo

              Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
              ================================================ FILE: test/test-pages/svg-parsing/expected-metadata.json ================================================ { "title": "SVG parsing", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\ntempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\nquis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\nconsequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\ncillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\nproident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/svg-parsing/expected.html ================================================

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/svg-parsing/source.html ================================================ SVG parsing

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/table-style-attributes/expected-metadata.json ================================================ { "title": "linux video", "byline": null, "dir": null, "excerpt": "linux usability\n ...or, why do I bother. © 2002, 2003\n Jamie Zawinski", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/table-style-attributes/expected.html ================================================

              linux usability
              ...or, why do I bother.
              © 2002, 2003 Jamie Zawinski

              In December 2002, I tried to install some software on my computer. The experience was, shall we say, less than pleasant. On many levels. I wrote about my experience, as I so often do.

              Then in January, the jackasses over at Slashdot posted a link to it, calling it a "review" of Linux video software. I guess you could consider it a review, if you were to squint at it just right. But really what it is is a rant about how I had an evening stolen from me by crap software design. It is a flame about the pathetic state of Linux usability in general, and the handful of video players I tried out in particular. It makes no attempt to be balanced or objective or exhaustive. It is a description of my experience. Perhaps your experience was different. Good for you.

              So of course that day I got hundreds of emails about it. Every Linux apologist in the world wanted to make sure I was fully informed of their opinion. The replies were roughly in the following groups:

              • "Right on! I had exactly the same experience! Thank you for putting it into words." (This was about 1/3 of the replies.)
              • "You're clearly an idiot, Linux is too sophisticated for you, you clearly are incapable of understanding anything, you should go back to kindergarten and/or use a Mac." (Oddly, all of these messages used the word `clearly' repeatedly.)
              • "If you don't like it, fix it yourself."
              • "Netscape sucks! XEmacs sucks! You suck! I never liked you anyway! And you swear too much!"
              • "How dare you criticize someone else's work! You got it for free! You should be on your knees thanking them for wasting your time!"
              • "While you have some valid complaints, I'm going to focus on this one inconsequential error you made in your characterization of one of the many roadblocks you encountered. You suck!"
              • "It's your fault for using Red Hat! You should be using Debian/Mandrake/Gentoo instead!"
              • "Red Hat 7.2 is totally obsolete! It's almost 14 months old! What were you expecting!"

              While I am flattered that so many logorrheic Linux fanboys are sufficiently interested in my opinions and experiences to share their deeply heartfelt views with me, you can all rest assured that:

                • I've heard it before; and
                • I didn't care the first time.

              So please. Don't bother sending me any more mail about this. It's a near certainty that I will just delete it unread, so you might as well not waste your time. Feel free to call me names on your own web page if you feel the need to get it out of your system. But kindly stay out of my inbox.

              that said...

              I understand that one can play videos on one's computer. I understand these videos come in many different formats. Every now and then I try to figure out what the Done Thing is, as far as playing movies on one's Linux machine.

              I finally found RPMs of mplayer that would consent to install themselves on a Red Hat 7.2 machine, and actually got it to play some videos. Amazing. But it's a total pain in the ass to use due to rampant "themeing." Why do people do this? They map this stupid shaped window with no titlebar (oh, sorry, your choice of a dozen stupidly-shaped windows without titlebars) all of which use fonts that are way too small to read. But, here's the best part, there's no way to raise the window to the top. So if another window ever gets on top of it, well, sorry, you're out of luck. And half of the themes always map the window at the very bottom of the screen -- conveniently under my panel where I can't reach it.

              Resizing the window changes the aspect ratio of the video! Yeah, I'm sure someone has ever wanted that.

              It moves the mouse to the upper left corner of every dialog box it creates! Which is great, because that means that when it gets into this cute little state of popping up a blank dialog that says "Error" five times a second, you can't even move the mouse over to another window to kill the program, you have to log in from another machine.

              Fucking morons.

              So I gave up on that, and tried to install gstreamer. Get this. Their propose ``solution'' for distributing binaries on Red Hat systems? They point you at an RPM that installs apt, the Debian package system! Yeah, that's a good idea, I want to struggle with two competing packaging systems on my machine just to install a single app. Well, I found some RPMs for Red Hat 7.2, but apparently they expect you to have already rectally inserted Gnome2 on that 7.2 system first. Uh, no. I've seen the horror of Red Hat 8.0, and there's no fucking way I'm putting Gnome2 on any more of my machines for at least another six months, maybe a year.

              Ok, no gstreamer. Let's try Xine. I found RPMs, and it sucks about the same as mplayer, and in about the same ways, though slightly less bad: it doesn't screw the aspect ratio when you resize the window; and at least its stupidly-shaped window is always forced to be on top. I don't like that either, but it's better than never being on top. It took me ten minutes to figure out where the "Open File" dialog was. It's on the button labeled "://" whose tooltip says "MRL Browser". Then you get to select file names from an oh-so-cute window that I guess is supposed to look like a tty, or maybe an LCD screen. It conveniently centers the file names in the list, and truncates them at about 30 characters. The scrollbar is also composed of "characters": it's an underscore.

              What are these fucktards thinking???

              Then I checked out Ogle again, and it hasn't been updated since the last time I tried, six months ago. It's a pretty decent DVD player, if you have the physical DVD. It does on-screen menus, and you can click on them with the mouse. But I don't need a DVD player (I have a hardware DVD player that works just fine.) It can't, as far as I can tell, play anything but actual discs.

              Oh, and even though I have libdvdcss installed (as evidenced by the fact that Ogle actually works) Xine won't play the same disc that Ogle will play. It seems to be claiming that the CSS stuff isn't installed, which it clearly is.

              An idiocy that all of these programs have in common is that, in addition to opening a window for the movie, and a window for the control panel, they also spray a constant spatter of curses crud on the terminal they were started from. I imagine at some point, there was some user who said, ``this program is pretty nice, but you know what it's missing? It's missing a lot of pointless chatter about what plugins and fonts have been loaded!''

              And here's the Random Commentary section:
              Makali wrote:
                Whenever a programmer thinks, "Hey, skins, what a cool idea", their computer's speakers should create some sort of cock-shaped soundwave and plunge it repeatedly through their skulls.

              I am fully in support of this proposed audio-cock technology.

              Various people wrote:

                You shouldn't even bother compiling the GUI into mplayer!

              So I should solve the problem of ``crappy GUI'' by replacing it with ``no GUI at all?'' I should use the program only from the command line, or by memorizing magic keystrokes? Awesome idea.

              Various other people wrote:

                You didn't try vlc!

              True, I hadn't. Now I have. It has an overly-complicated UI, (the Preferences panel is a festival of overkill) but at least it uses standard menus and buttons, so it doesn't make you want to claw your eyes out immediately. But, it can only play a miniscule number of video formats, so it's mostly useless. *plonk*

              Someone else wrote:

                Have you considered changing distributions?

              Yes, every single time I try something like this, I very seriously consider getting a Mac.

              Really the only thing that's stopping me is that I fear the Emacs situation.

              (By which I mean, ``Lack of a usable version thereof.'' No, running RMSmacs inside a terminal window doesn't qualify. Nor does running an X server on the Mac: if I were going to switch, why in the world would I continue inflicting the X Windows Disaster on myself? Wouldn't getting away from that be the whole point?)

                (I understand there is an almost-functional Aqua version of RMSmacs now. I'll probably check it out at some point, but the problem with me switching from XEmacs to RMSmacs is that it would probably result in another Slashdork post, meaning I'd wake up to another 150+ poorly spelled flames in my inbox... I'm hoping for a Aquafied XEmacs, but I know that's not likely to happen any time soon.)

              By the way, the suggestion to switch Linux distrubutions in order to get a single app to work might sound absurd at first. And that's because it is. But I've been saturated with Unix-peanut-gallery effluvia for so long that it no longer even surprises me when every question -- no matter how simple -- results in someone suggesting that you either A) patch your kernel or B) change distros. It's inevitable and inescapable, like Hitler.

              [ up ]

              ================================================ FILE: test/test-pages/table-style-attributes/source.html ================================================ linux video

              linux usability
              ...or, why do I bother.
              © 2002, 2003 Jamie Zawinski


              In December 2002, I tried to install some software on my computer. The experience was, shall we say, less than pleasant. On many levels. I wrote about my experience, as I so often do.

              Then in January, the jackasses over at Slashdot posted a link to it, calling it a "review" of Linux video software. I guess you could consider it a review, if you were to squint at it just right. But really what it is is a rant about how I had an evening stolen from me by crap software design. It is a flame about the pathetic state of Linux usability in general, and the handful of video players I tried out in particular. It makes no attempt to be balanced or objective or exhaustive. It is a description of my experience. Perhaps your experience was different. Good for you.

              So of course that day I got hundreds of emails about it. Every Linux apologist in the world wanted to make sure I was fully informed of their opinion. The replies were roughly in the following groups:

              • "Right on! I had exactly the same experience! Thank you for putting it into words." (This was about 1/3 of the replies.)

              • "You're clearly an idiot, Linux is too sophisticated for you, you clearly are incapable of understanding anything, you should go back to kindergarten and/or use a Mac." (Oddly, all of these messages used the word `clearly' repeatedly.)

              • "If you don't like it, fix it yourself."

              • "Netscape sucks! XEmacs sucks! You suck! I never liked you anyway! And you swear too much!"

              • "How dare you criticize someone else's work! You got it for free! You should be on your knees thanking them for wasting your time!"

              • "While you have some valid complaints, I'm going to focus on this one inconsequential error you made in your characterization of one of the many roadblocks you encountered. You suck!"

              • "It's your fault for using Red Hat! You should be using Debian/Mandrake/Gentoo instead!"

              • "Red Hat 7.2 is totally obsolete! It's almost 14 months old! What were you expecting!"

              While I am flattered that so many logorrheic Linux fanboys are sufficiently interested in my opinions and experiences to share their deeply heartfelt views with me, you can all rest assured that:

                • I've heard it before; and
                • I didn't care the first time.

              So please. Don't bother sending me any more mail about this. It's a near certainty that I will just delete it unread, so you might as well not waste your time. Feel free to call me names on your own web page if you feel the need to get it out of your system. But kindly stay out of my inbox.


              that said...

              I understand that one can play videos on one's computer. I understand these videos come in many different formats. Every now and then I try to figure out what the Done Thing is, as far as playing movies on one's Linux machine.

              I finally found RPMs of mplayer that would consent to install themselves on a Red Hat 7.2 machine, and actually got it to play some videos. Amazing. But it's a total pain in the ass to use due to rampant "themeing." Why do people do this? They map this stupid shaped window with no titlebar (oh, sorry, your choice of a dozen stupidly-shaped windows without titlebars) all of which use fonts that are way too small to read. But, here's the best part, there's no way to raise the window to the top. So if another window ever gets on top of it, well, sorry, you're out of luck. And half of the themes always map the window at the very bottom of the screen -- conveniently under my panel where I can't reach it.

              Resizing the window changes the aspect ratio of the video! Yeah, I'm sure someone has ever wanted that.

              It moves the mouse to the upper left corner of every dialog box it creates! Which is great, because that means that when it gets into this cute little state of popping up a blank dialog that says "Error" five times a second, you can't even move the mouse over to another window to kill the program, you have to log in from another machine.

              Fucking morons.

              So I gave up on that, and tried to install gstreamer. Get this. Their propose ``solution'' for distributing binaries on Red Hat systems? They point you at an RPM that installs apt, the Debian package system! Yeah, that's a good idea, I want to struggle with two competing packaging systems on my machine just to install a single app. Well, I found some RPMs for Red Hat 7.2, but apparently they expect you to have already rectally inserted Gnome2 on that 7.2 system first. Uh, no. I've seen the horror of Red Hat 8.0, and there's no fucking way I'm putting Gnome2 on any more of my machines for at least another six months, maybe a year.

              Ok, no gstreamer. Let's try Xine. I found RPMs, and it sucks about the same as mplayer, and in about the same ways, though slightly less bad: it doesn't screw the aspect ratio when you resize the window; and at least its stupidly-shaped window is always forced to be on top. I don't like that either, but it's better than never being on top. It took me ten minutes to figure out where the "Open File" dialog was. It's on the button labeled "://" whose tooltip says "MRL Browser". Then you get to select file names from an oh-so-cute window that I guess is supposed to look like a tty, or maybe an LCD screen. It conveniently centers the file names in the list, and truncates them at about 30 characters. The scrollbar is also composed of "characters": it's an underscore.

              What are these fucktards thinking???

              Then I checked out Ogle again, and it hasn't been updated since the last time I tried, six months ago. It's a pretty decent DVD player, if you have the physical DVD. It does on-screen menus, and you can click on them with the mouse. But I don't need a DVD player (I have a hardware DVD player that works just fine.) It can't, as far as I can tell, play anything but actual discs.

              Oh, and even though I have libdvdcss installed (as evidenced by the fact that Ogle actually works) Xine won't play the same disc that Ogle will play. It seems to be claiming that the CSS stuff isn't installed, which it clearly is.

              An idiocy that all of these programs have in common is that, in addition to opening a window for the movie, and a window for the control panel, they also spray a constant spatter of curses crud on the terminal they were started from. I imagine at some point, there was some user who said, ``this program is pretty nice, but you know what it's missing? It's missing a lot of pointless chatter about what plugins and fonts have been loaded!''

              And here's the Random Commentary section:

              Makali wrote:

              I am fully in support of this proposed audio-cock technology.

              Various people wrote:

              So I should solve the problem of ``crappy GUI'' by replacing it with ``no GUI at all?'' I should use the program only from the command line, or by memorizing magic keystrokes? Awesome idea.

              Various other people wrote:

              True, I hadn't. Now I have. It has an overly-complicated UI, (the Preferences panel is a festival of overkill) but at least it uses standard menus and buttons, so it doesn't make you want to claw your eyes out immediately. But, it can only play a miniscule number of video formats, so it's mostly useless. *plonk*

              Someone else wrote:

              Yes, every single time I try something like this, I very seriously consider getting a Mac.

              Really the only thing that's stopping me is that I fear the Emacs situation.

              (By which I mean, ``Lack of a usable version thereof.'' No, running RMSmacs inside a terminal window doesn't qualify. Nor does running an X server on the Mac: if I were going to switch, why in the world would I continue inflicting the X Windows Disaster on myself? Wouldn't getting away from that be the whole point?)

              By the way, the suggestion to switch Linux distrubutions in order to get a single app to work might sound absurd at first. And that's because it is. But I've been saturated with Unix-peanut-gallery effluvia for so long that it no longer even surprises me when every question -- no matter how simple -- results in someone suggesting that you either A) patch your kernel or B) change distros. It's inevitable and inescapable, like Hitler.

              [ up ]

              ================================================ FILE: test/test-pages/telegraph/expected-metadata.json ================================================ { "title": "Zimbabwe coup: Robert Mugabe and wife Grace 'insisting he finishes his term', as priest steps in to mediate", "byline": null, "dir": null, "lang": "en-GB", "excerpt": "Zimbabwe President Robert Mugabe, his wife Grace and two key figures from her G40 political faction are under house arrest at Mugabe's \"Blue House\" compound in Harare and are insisting the 93 year-old finishes his presidential term, a source said.", "siteName": "The Telegraph", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/telegraph/expected.html ================================================

              Zimbabwe President Robert Mugabe, his wife Grace and two key figures from her G40 political faction are under house arrest at Mugabe's "Blue House" compound in Harare and are insisting the 93 year-old finishes his presidential term, a source said.

              The G40 figures are cabinet ministers Jonathan Moyo and Saviour Kasukuwere, who fled to the compound after their homes were attacked by troops in Tuesday night's coup, the source, who said he had spoken to people inside the compound, told Reuters.

              Mr Mugabe is resisting mediation by a Catholic priest to allow the former guerrilla a graceful exit after the military takeover.

              The priest, Fidelis Mukonori, is acting as a middle-man between Mr Mugabe and the generals, who seized power in a targeted operation against "criminals" in his entourage, a senior political source told Reuters.

              The source could not provide details of the talks, which appear to be aimed at a smooth and bloodless transition after the departure of Mr Mugabe, who has led Zimbabwe since independence in 1980.

              Mr Mugabe, still seen by many Africans as a liberation hero, is reviled in the West as a despot whose disastrous handling of the economy and willingness to resort to violence to maintain power destroyed one of Africa's most promising states.

              Zimbabwean intelligence reports seen by Reuters suggest that former security chief Emmerson Mnangagwa, who was ousted as vice-president this month, has been mapping out a post-Mugabe vision with the military and opposition for more than a year.

              Fuelling speculation that Mnangagwa's plan might be rolling into action, opposition leader Morgan Tsvangirai, who has been receiving cancer treatment in Britain and South Africa, returned to Harare late on Wednesday, his spokesman said.

              South Africa said Mr Mugabe had told President Jacob Zuma by telephone on Wednesday that he was confined to his home but was otherwise fine and the military said it was keeping him and his family, including wife Grace, safe.

              Despite the lingering admiration for Mr Mugabe, there is little public affection for 52-year-old Grace, a former government typist who started having an affair with Mr Mugabe in the early 1990s as his first wife, Sally, was dying of kidney disease.

              Dubbed "DisGrace" or "Gucci Grace" on account of her reputed love of shopping, she enjoyed a meteoric rise through the ranks of Mugabe's ruling Zanu-PF in the last two years, culminating in Mnangagwa's removal a week ago - a move seen as clearing the way for her to succeed her husband.

              In contrast to the high political drama unfolding behind closed doors, the streets of the capital remained calm, with people going about their daily business, albeit under the watch of soldiers on armoured vehicles at strategic locations.

              Whatever the final outcome, the events could signal a once-in-a-generation change for the former British colony, a regional breadbasket reduced to destitution by economic policies Mr Mugabe's critics have long blamed on him.

              ================================================ FILE: test/test-pages/telegraph/source.html ================================================ Zimbabwe coup: Robert Mugabe and wife Grace 'insisting he finishes his term', as priest steps in to mediate
              The Telegraph
              The Telegraph

              Zimbabwe coup: Robert Mugabe and wife Grace 'insisting he finishes his term', as priest steps in to mediate

              Zimbabwe president Robert Mugabe with ZDF commander general Constantino Chiwenga at State House in an image the Harare Herald claims was taken on November 16 - the day after the military takeover Credit: HARARE HERALD

              Zimbabwe President Robert Mugabe, his wife Grace and two key figures from her G40 political faction are under house arrest at Mugabe's "Blue House" compound in Harare and are insisting the 93 year-old finishes his presidential term, a source said.

              The G40 figures are cabinet ministers Jonathan Moyo and Saviour Kasukuwere, who fled to the compound after their homes were attacked by troops in Tuesday night's coup, the source, who said he had spoken to people inside the compound, told Reuters.

              Mr Mugabe is resisting mediation by a Catholic priest to allow the former guerrilla a graceful exit after the military takeover.

              The priest, Fidelis Mukonori, is acting as a middle-man between Mr Mugabe and the generals, who seized power in a targeted operation against "criminals" in his entourage, a senior political source told Reuters.

              The source could not provide details of the talks, which appear to be aimed at a smooth and bloodless transition after the departure of Mr Mugabe, who has led Zimbabwe since independence in 1980.

              Mr Mugabe, still seen by many Africans as a liberation hero, is reviled in the West as a despot whose disastrous handling of the economy and willingness to resort to violence to maintain power destroyed one of Africa's most promising states.
              Zimbabwean opposition leader Morgan Tsvangirai, right, meets church leaders Bishop Trevor Manhanga, centre, and Father Fidelis Mukonori in 2006. Father Mukonori is said to be mediating in the current crisis Credit:  DESMOND KWANDE/ AFP

              Zimbabwean intelligence reports seen by Reuters suggest that former security chief Emmerson Mnangagwa, who was ousted as vice-president this month, has been mapping out a post-Mugabe vision with the military and opposition for more than a year.

              Fuelling speculation that Mnangagwa's plan might be rolling into action, opposition leader Morgan Tsvangirai, who has been receiving cancer treatment in Britain and South Africa, returned to Harare late on Wednesday, his spokesman said.

              South Africa said Mr Mugabe had told President Jacob Zuma by telephone on Wednesday that he was confined to his home but was otherwise fine and the military said it was keeping him and his family, including wife Grace, safe.

              Despite the lingering admiration for Mr Mugabe, there is little public affection for 52-year-old Grace, a former government typist who started having an affair with Mr Mugabe in the early 1990s as his first wife, Sally, was dying of kidney disease.

              Dubbed "DisGrace" or "Gucci Grace" on account of her reputed love of shopping, she enjoyed a meteoric rise through the ranks of Mugabe's ruling Zanu-PF in the last two years, culminating in Mnangagwa's removal a week ago - a move seen as clearing the way for her to succeed her husband.
              A man walks past an armoured personnel carrier parked on a Harare street on Thursday Credit: STR/AFP 

              In contrast to the high political drama unfolding behind closed doors, the streets of the capital remained calm, with people going about their daily business, albeit under the watch of soldiers on armoured vehicles at strategic locations.

              Whatever the final outcome, the events could signal a once-in-a-generation change for the former British colony, a regional breadbasket reduced to destitution by economic policies Mr Mugabe's critics have long blamed on him.

              ================================================ FILE: test/test-pages/theverge/expected-metadata.json ================================================ { "title": "Apple’s Vision Pro hands-on: the Retina display moment for headsets", "byline": "Alex Heath", "dir": null, "lang": "en", "excerpt": "I tried Apple’s new Vision Pro headset, and just like the introduction of the iPhone 4 over a decade ago, there’s no going back from here.", "siteName": "The Verge", "publishedTime": "2023-06-07T20:54:26.829Z", "readerable": true } ================================================ FILE: test/test-pages/theverge/expected.html ================================================

              I still remember using the iPhone 4 for the first time in 2010. That was when Apple shipped its first-ever Retina display and Steve Jobs said that, once you use it, “you can’t go back.” It was something I couldn’t unsee, like looking through prescription glasses for the first time.

              That’s exactly how I felt after a demo of the Apple Vision Pro yesterday at the company’s headquarters in Cupertino, California. A computer you strap to your face should be primarily judged not only by what you can do with it but also by the quality of what you can see through it. The Vision Pro blows away every other headset in this regard. It’s the industry’s Retina display moment. There’s no going back.

              The headset packs an insane 23 megapixels into dual MicroOLED panels, meaning that each eye looks through a roughly 4K display. The $1,000 Meta Quest Pro, by contrast, has a resolution of 1800 x 1920 per eye. For those who need vision correction like me, Apple has partnered with Zeiss to sell prescription inserts that clip onto the inner-facing displays. That helps make the headset not only thinner but, in my experience, also much more comfortable to wear. 

              After scanning my face and ears on an iPhone to calibrate the device to my head (the experience will be immediately familiar to anyone who has set up Face ID), I handed my glasses to an optometrist Apple had on-site to have my prescription made for the demo. After a few minutes, I was whisked into another brightly lit, temperature-controlled room with a headset that had my prescription inserted.

              The Vision Pro displaying a window floating in a room.

              I wasn’t allowed to record or capture footage of my Vision Pro demo, so this photo from Apple will have to do.

              Visually, the most memorable experience of my roughly 30-minute, highly controlled demo wasn’t the butterfly landing on my finger, the 3D Avatar clip, or even viewing Apple’s new surreal 3D photos and videos. It was when I had three windows open at once for Messages, Safari, and Photos. I used the headset’s eye tracking and pinching gesture to quickly (and I do mean quickly — navigating this way is incredibly intuitive) place each window at a different depth in the room.

              I placed Messages to my immediate right and almost uncomfortably close to my face, Safari in the middle of the room, and Photos against the wall I was facing. I could see Messages up close just as clearly as the text in the Photos app more than eight feet away. There were no discernible pixels anywhere.

              My colleague David Pierce, who also tried the Vision Pro this week, has already pointed out that it will be a great TV. I agree and could see myself actually watching a movie in it, which is something I’d never say about the other headsets out there. It’s clear that Apple chose not to compromise on the quality of the visual experience, even if it means that buying the Vision Pro with prescription inserts will likely cost as much as a used Ford Focus.

              There’s plenty about the Vision Pro that remains to be seen, namely the front face display that can show where the wearer’s eyes are looking. It wasn’t turned on for my demo and appears to not be finished. I wasn’t allowed to use the virtual keyboard, and the ability to take what Apple is calling “spatial” photos and videos through a dedicated button on the headset wasn’t enabled. I fully expect the software I experienced to change a lot before the device ships next year, which makes it easiest to judge the Vision Pro on what won’t change. 

              It’s easiest to judge the Vision Pro on what won’t change

              Apple’s goal for the Vision Pro is clearly to get developers building for the headset and figuring out its killer apps for future, cheaper versions, so not compromising on the display initially is the right call. It’s a compelling enough leap forward for headset optics alone to make the device worth trying. Like that first Retina display, it’s something you can’t unsee.

              Sign up for Command Line, a paid weekly newsletter from Alex Heath about the tech industry’s inside conversation. Your first month is free!

              Monthly

              $7/month

              A flexible plan you can cancel anytime.

              Annual

              $70/year

              A discounted plan to keep you up to date all year.

              Corporate

              $60/person/year

              Keep your team informed on the inside conversation.

              We accept credit card, Apple Pay and Google Pay.

              ================================================ FILE: test/test-pages/theverge/source.html ================================================ Apple’s Vision Pro hands-on: the Retina display moment for headsets - The Verge
              Skip to main content
              The Verge The Verge logo. The Verge homepage
              The Verge The Verge logo.

              Apple’s Vision Pro is the Retina display moment for headsets

              I tried the Vision Pro, and just like the introduction of the iPhone 4 over a decade ago, there’s no going back from here.

              By Alex Heath, a deputy editor and author of the Command Line newsletter. He’s covered the tech industry for over a decade at The Information and other outlets.

              |

              Share this story

              The Apple Vision Pro headset on display at Apple’s headquarters in Cupertino.
              The Apple Vision Pro.
              Image: Vjeran Pavic / The Verge

              I still remember using the iPhone 4 for the first time in 2010. That was when Apple shipped its first-ever Retina display and Steve Jobs said that, once you use it, “you can’t go back.” It was something I couldn’t unsee, like looking through prescription glasses for the first time.

              That’s exactly how I felt after a demo of the Apple Vision Pro yesterday at the company’s headquarters in Cupertino, California. A computer you strap to your face should be primarily judged not only by what you can do with it but also by the quality of what you can see through it. The Vision Pro blows away every other headset in this regard. It’s the industry’s Retina display moment. There’s no going back.

              The headset packs an insane 23 megapixels into dual MicroOLED panels, meaning that each eye looks through a roughly 4K display. The $1,000 Meta Quest Pro, by contrast, has a resolution of 1800 x 1920 per eye. For those who need vision correction like me, Apple has partnered with Zeiss to sell prescription inserts that clip onto the inner-facing displays. That helps make the headset not only thinner but, in my experience, also much more comfortable to wear. 

              After scanning my face and ears on an iPhone to calibrate the device to my head (the experience will be immediately familiar to anyone who has set up Face ID), I handed my glasses to an optometrist Apple had on-site to have my prescription made for the demo. After a few minutes, I was whisked into another brightly lit, temperature-controlled room with a headset that had my prescription inserted.

              The Vision Pro displaying a window floating in a room.
              I wasn’t allowed to record or capture footage of my Vision Pro demo, so this photo from Apple will have to do.

              Visually, the most memorable experience of my roughly 30-minute, highly controlled demo wasn’t the butterfly landing on my finger, the 3D Avatar clip, or even viewing Apple’s new surreal 3D photos and videos. It was when I had three windows open at once for Messages, Safari, and Photos. I used the headset’s eye tracking and pinching gesture to quickly (and I do mean quickly — navigating this way is incredibly intuitive) place each window at a different depth in the room.

              I placed Messages to my immediate right and almost uncomfortably close to my face, Safari in the middle of the room, and Photos against the wall I was facing. I could see Messages up close just as clearly as the text in the Photos app more than eight feet away. There were no discernible pixels anywhere.

              Related

              My colleague David Pierce, who also tried the Vision Pro this week, has already pointed out that it will be a great TV. I agree and could see myself actually watching a movie in it, which is something I’d never say about the other headsets out there. It’s clear that Apple chose not to compromise on the quality of the visual experience, even if it means that buying the Vision Pro with prescription inserts will likely cost as much as a used Ford Focus.

              There’s plenty about the Vision Pro that remains to be seen, namely the front face display that can show where the wearer’s eyes are looking. It wasn’t turned on for my demo and appears to not be finished. I wasn’t allowed to use the virtual keyboard, and the ability to take what Apple is calling “spatial” photos and videos through a dedicated button on the headset wasn’t enabled. I fully expect the software I experienced to change a lot before the device ships next year, which makes it easiest to judge the Vision Pro on what won’t change. 

              It’s easiest to judge the Vision Pro on what won’t change

              Apple’s goal for the Vision Pro is clearly to get developers building for the headset and figuring out its killer apps for future, cheaper versions, so not compromising on the display initially is the right call. It’s a compelling enough leap forward for headset optics alone to make the device worth trying. Like that first Retina display, it’s something you can’t unsee.

              More from this stream WWDC 2023 news: Apple Vision Pro, Mac Pro, iOS 17, and more

              See all 121 stories
              ================================================ FILE: test/test-pages/title-and-h1-discrepancy/expected-metadata.json ================================================ { "title": "This is a long title with a colon: Hello there", "byline": null, "dir": null, "excerpt": "Lorem\n ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/title-and-h1-discrepancy/expected.html ================================================

              This is a long title with a colon: But the final text here is different

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/title-and-h1-discrepancy/source.html ================================================ This is a long title with a colon: Hello there

              This is a long title with a colon: But the final text here is different

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
              ================================================ FILE: test/test-pages/title-en-dash/expected-metadata.json ================================================ { "title": "Lorem ipsum dolor sit amet", "byline": null, "dir": null, "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/title-en-dash/expected.html ================================================

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

              ================================================ FILE: test/test-pages/title-en-dash/source.html ================================================ Lorem ipsum dolor sit amet – My website
              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
              Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
              ================================================ FILE: test/test-pages/tmz-1/expected-metadata.json ================================================ { "title": "Lupita Nyong'o's $150K Pearl Oscar Dress -- STOLEN!!!", "byline": null, "dir": null, "excerpt": "Lupita Nyong'o's now-famous Oscar dress -- adorned in pearls -- was stolen right out of her hotel room ... TMZ has learned. Law enforcement sources tell…", "siteName": "http://www.tmz.com", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/tmz-1/expected.html ================================================

              $150K Pearl Oscar Dress ... STOLEN!!!!

              2/26/2015 7:11 AM PST BY TMZ STAFF

              EXCLUSIVE

              0225-lupita-nyongo-getty-01Lupita Nyong'o's now-famous Oscar dress -- adorned in pearls -- was stolen right out of her hotel room ... TMZ has learned.

              Law enforcement sources tell TMZ ... the dress was taken out of Lupita's room at The London West Hollywood. The dress is made of pearls ... 6,000 white Akoya pearls. It's valued at $150,000.

              Our sources say Lupita told cops it was taken from her room sometime between 8 AM and 9 PM Wednesday ... while she was gone.  

              We're told there is security footage that cops are looking at that could catch the culprit right in the act. 

              update_graphic_red_bar12:00 PM PT -- Sheriff's deputies were at The London Thursday morning.  We know they were in the manager's office and we're told they have looked at security footage to determine if they can ID the culprit.

              0226-SUB-london-hotel-swipe-tmz-02

              ================================================ FILE: test/test-pages/tmz-1/source.html ================================================ Lupita Nyong'o's $150K Pearl Oscar Dress -- STOLEN!!! | TMZ.com 
              TMZ

              Our TV Shows

                  Got a Tip?

                  Call TMZ at (888) 847-9869 or Click Here

                  Home

                  Lupita Nyong'o's $150K Pearl Oscar Dress -- STOLEN!!!

                  Lupita Nyong'o

                  $150K Pearl Oscar Dress ... STOLEN!!!!

                  2/26/2015 7:11 AM PST BY TMZ STAFF
                  EXCLUSIVE

                  0225-lupita-nyongo-getty-01Lupita Nyong'o's now-famous Oscar dress -- adorned in pearls -- was stolen right out of her hotel room ... TMZ has learned.

                  Law enforcement sources tell TMZ ... the dress was taken out of Lupita's room at The London West Hollywood. The dress is made of pearls ... 6,000 white Akoya pearls. It's valued at $150,000.

                  Our sources say Lupita told cops it was taken from her room sometime between 8 AM and 9 PM Wednesday ... while she was gone.  

                  We're told there is security footage that cops are looking at that could catch the culprit right in the act. 

                  update_graphic_red_bar12:00 PM PT -- Sheriff's deputies were at The London Thursday morning.  We know they were in the manager's office and we're told they have looked at security footage to determine if they can ID the culprit.

                  0226-SUB-london-hotel-swipe-tmz-02

                  See also

                  Around The Web

                  ================================================ FILE: test/test-pages/toc-missing/expected-metadata.json ================================================ { "title": "Simple Anomaly Detection Using Plain SQL", "byline": "Haki Benita", "dir": null, "lang": "en", "excerpt": "Many developers think that having a critical bug in their code is the worse thing that can happen. Well, there is something much worst than that: Having a critical bug in your code and not knowing about it! Using some high school level statistics and a fair knowledge of SQL, I implemented a very simple anomaly detection system.", "siteName": "Haki Benita", "publishedTime": "2020-09-21", "readerable": true } ================================================ FILE: test/test-pages/toc-missing/expected.html ================================================

                  Many developers think that having a critical bug in their code is the worst thing that can happen. Well, there is something much worse than that: Having a critical bug in your code and not knowing about it!

                  To make sure I get notified about critical bugs as soon as possible, I started looking for ways to find anomalies in my data. I quickly found that information about these subjects tend to get very complicated, and involve a lot of ad-hoc tools and dependencies.

                  I'm not a statistician and not a data scientist, I'm just a developer. Before I introduce dependencies into my system I make sure I really can't do without them. So, using some high school level statistics and a fair knowledge of SQL, I implemented a simple anomaly detection system that works.

                  Can you spot the anomaly?<br><small>Photo by <a href="https://unsplash.com/photos/KmKZV8pso-s">Ricardo Gomez Angel</a></small>
                  Can you spot the anomaly?
                  Photo by Ricardo Gomez Angel
                  Table of Contents

                  Interactive Editor

                  Detecting Anomalies

                  Anomaly in a data series is a significant deviation from some reasonable value. Looking at this series of numbers for example, which number stands out? 

                  2, 3, 5, 2, 3, 12, 5, 3, 4

                  The number that stands out in this series is 12.

                  Scatter plot
                  Scatter plot

                  This is intuitive to a human, but computer programs don't have intuition...

                  To find the anomaly in the series we first need to define what a reasonable value is, and then define how far away from this value we consider a significant deviation. A good place to start looking for a reasonable value is the mean: 

                  SELECT avg(n)
FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

       avg
────────────────────
4.3333333333333333

                  The mean is ~4.33.

                  Next, we need to define the deviation. Let's use Standard Deviation: 

                  SELECT stddev(n)
FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

      stddev
────────────────────
3.0822070014844882

                  Standard deviation is the square root of the variance, which is the average squared distance from the mean. In this case it's 3.08.

                  Now that we've defined a "reasonable" value and a deviation, we can define a range of acceptable values: 

                  SELECT
   avg(n) - stddev(n) AS lower_bound,
   avg(n) + stddev(n) AS upper_bound
FROM
   unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

    lower_bound    │     upper_bound
───────────────────┼────────────────────
1.2511263318488451 │ 7.4155403348178215

                  The range we defined is one standard deviation from the mean. Any value outside this range is considered an anomaly: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
bounds AS (
   SELECT
       avg(n) - stddev(n) AS lower_bound,
       avg(n) + stddev(n) AS upper_bound
   FROM
       series
)
SELECT
   n,
   n NOT BETWEEN lower_bound AND upper_bound AS is_anomaly
FROM
   series,
   bounds;

n  │ is_anomaly
───┼────────────
 2 │ f
 3 │ f
 5 │ f
 2 │ f
 3 │ f
12 │ t
 5 │ f
 3 │ f
 4 │ f

                  Using the query we found that the value 12 is outside the range of acceptable values, and identified it as an anomaly.

                  Understanding Z-Score

                  Another way to represent a range of acceptable values is using a z-score. z-score, or Standard Score, is the number of standard deviations from the mean. In the previous section, our acceptable range was one standard deviation from the mean, or in other words, a z-score in the range ±1: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_mean,
       stddev(n) as series_stddev
   FROM
       series
)
SELECT
   n,
   (n - series_mean) / series_stddev as zscore
FROM
   series,
   stats;

n  │         zscore
───┼─────────────────────────
 2 │ -0.75703329861022517346
 3 │ -0.43259045634870009448
 5 │  0.21629522817435006346
 2 │ -0.75703329861022517346
 3 │ -0.43259045634870009448
12 │      2.4873951240050256
 5 │  0.21629522817435006346
 3 │ -0.43259045634870009448
 4 │ -0.10814761408717501551

                  Like before, we can detect anomalies by searching for values which are outside the acceptable range using the z-score: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_avg,
       stddev(n) as series_stddev
   FROM
       series
),
zscores AS (
   SELECT
       n,
       (n - series_avg) / series_stddev AS zscore
   FROM
       series,
       stats
)
SELECT
   *,
   zscore NOT BETWEEN -1 AND 1 AS is_anomaly
FROM
   zscores;

n  │         zscore          │ is_anomaly
───┼─────────────────────────┼────────────
 2 │ -0.75703329861022517346 │ f
 3 │ -0.43259045634870009448 │ f
 5 │  0.21629522817435006346 │ f
 2 │ -0.75703329861022517346 │ f
 3 │ -0.43259045634870009448 │ f
12 │      2.4873951240050256 │ t
 5 │  0.21629522817435006346 │ f
 3 │ -0.43259045634870009448 │ f
 4 │ -0.10814761408717501551 │ f

                  Using z-score, we also identified 12 as an anomaly in this series.

                  Optimizing Z-Score

                  So far we used one standard deviation from the mean, or a z-score of ±1 to identify anomalies. Changing the z-score threshold can affect our results. For example, let's see what anomalies we identify when the z-score is greater than 0.5 and when it's greater than 3: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_avg,
       stddev(n) as series_stddev
   FROM
       series
),
zscores AS (
   SELECT
       n,
       (n - series_avg) / series_stddev AS zscore
   FROM
       series,
       stats
)
SELECT
   *,
   zscore NOT BETWEEN -0.5 AND 0.5 AS is_anomaly_0_5,
   zscore NOT BETWEEN -1 AND 1 AS is_anomaly_1,
   zscore NOT BETWEEN -3 AND 3 AS is_anomaly_3
FROM
   zscores;

n  │         zscore          │ is_anomaly_0_5 │ is_anomaly_1 │ is_anomaly_3
───┼─────────────────────────┼────────────────┼──────────────┼──────────────
 2 │ -0.75703329861022517346 │ t              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
 5 │  0.21629522817435006346 │ f              │ f            │ f
 2 │ -0.75703329861022517346 │ t              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
12 │      2.4873951240050256 │ t              │ t            │ f
 5 │  0.21629522817435006346 │ f              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
 4 │ -0.10814761408717501551 │ f              │ f            │ f

                  Let's see what we got:

                  • When we decreased the z-score threshold to 0.5, we identified the value 2 as an anomaly in addition to the value 12.
                  • When we increased the z-score threshold to 3 we did not identify any anomaly.

                  The quality of our results are directly related to the parameters we set for the query. Later we'll see how using backtesting can help us identify ideal values.

                  Analyzing a Server Log

                  Application servers such as nginx, Apache and IIS write a lot of useful information to access logs. The data in these logs can be extremely useful in identifying anomalies.

                  We are going to analyze logs of a web application, so the data we are most interested in is the timestamp and the status code of every response from the server. To illustrate the type of insight we can draw from just this data:

                  • A sudden increase in 500 status code: You may have a problem in the server. Did you just push a new version? Is there an external service you're using that started failing in unexpected ways?
                  • A sudden increase in 400 status code: You may have a problem in the client. Did you change some validation logic and forgot to update the client? Did you make a change and forgot to handle backward compatibility?
                  • A sudden increase in 404 status code: You may have an SEO problem. Did you move some pages and forgot to set up redirects? Is there some script kiddy running a scan on your site?
                  • A sudden increase in 200 status code: You either have some significant legit traffic coming in, or you are under a DOS attack. Either way, you probably want to check where it's coming from.

                  Preparing the Data

                  Parsing and processing logs is outside the scope of this article, so let's assume we did that and we have a table that looks like this: 

                  CREATE TABLE server_log_summary AS (
   period timestamptz,
   status_code int,
   entries int
);

                  The table stores the number of entries for each status code at a given period. For example, our table stores how many responses returned each status code every minute: 

                  db=# SELECT * FROM server_log_summary ORDER BY period DESC LIMIT 10;

        period         │ status_code │ entries
───────────────────────┼─────────────┼─────────
2020-08-01 18:00:00+00 │         200 │    4084
2020-08-01 18:00:00+00 │         404 │       0
2020-08-01 18:00:00+00 │         400 │      24
2020-08-01 18:00:00+00 │         500 │       0
2020-08-01 17:59:00+00 │         400 │      12
2020-08-01 17:59:00+00 │         200 │    3927
2020-08-01 17:59:00+00 │         500 │       0
2020-08-01 17:59:00+00 │         404 │       0
2020-08-01 17:58:00+00 │         400 │       2
2020-08-01 17:58:00+00 │         200 │    3850

                  Note that the table has a row for every minute, even if the status code was never returned in that minute. Given a table of statuses, it's very tempting to do something like this: 

                  -- Wrong!
SELECT
   date_trunc('minute', timestamp) AS period,
   status_code,
   count(*) AS entries
FROM
   server_log
GROUP BY
   period,
   status_code;

                  This is a common mistake and it can leave you with gaps in the data. Zero is a value, and it holds a significant meaning. A better approach is to create an "axis", and join to it: 

                  -- Correct!
WITH axis AS (
   SELECT
       status_code,
       generate_series(
           date_trunc('minute', now()),
           date_trunc('minute', now() - interval '1 hour'),
           interval '1 minute' * -1
       ) AS period
   FROM (
       VALUES (200), (400), (404), (500)
   ) AS t(status_code)
)
SELECT
   a.period,
   a.status_code,
   count(*) AS entries
FROM
   axis a
   LEFT JOIN server_log l ON (
       date_trunc('minute', l.timestamp) = a.period
       AND l.status_code = a.status_code
   )
GROUP BY
   period,
   status_code;

                  First we generate an axis using a cartesian join between the status codes we want to track, and the times we want to monitor. To generate the axis we used two nice features of PostgreSQL:

                  • generate_series: function that generates a range of values.
                  • VALUES list: special clause that can generate "constant tables", as the documentation calls it. You might be familiar with the VALUES clause from INSERT statements. In the old days, to generate data we had to use a bunch of SELECT ... UNION ALL... using VALUES is much nicer.

                  After generating the axis, we left join the actual data into it to get a complete series for each status code. The resulting data has no gaps, and is ready for analysis.

                  Getting a Sense of the Data

                  To get a sense of the data, let's draw a stacked bar chart by status:

                  stacked bar chart by status, over time
                  stacked bar chart by status, over time

                  The chart shows a period of 12 hours. It looks like we have a nice trend with two peaks at around 09:30 and again at 18:00.

                  We also spot right away that at ~11:30 there was a significant increase in 500 errors. The burst died down after around 10 minutes. This is the type of anomalies we want to identify early on.

                  It's entirely possible that there were other problems during that time, we just can't spot them with a naked eye.

                  Identifying Anomalies

                  In anomaly detection systems, we usually want to identify if we have an anomaly right now, and send an alert.

                  To identify if the last datapoint is an anomaly, we start by calculating the mean and standard deviation for each status code in the past hour: 

                  db=# WITH stats AS (
   SELECT
       status_code,
       (MAX(ARRAY[EXTRACT('epoch' FROM period), entries]))[2] AS last_value,
       AVG(entries) AS mean_entries,
       STDDEV(entries) AS stddev_entries
   FROM
       server_log_summary
   WHERE
       -- In the demo data use:
       -- period > '2020-08-01 17:00 UTC'::timestamptz
       period > now() - interval '1 hour'
   GROUP BY
       status_code
)
SELECT * FROM stats;

status_code │ last_value │      mean_entries      │     stddev_entries
────────────┼────────────┼────────────────────────┼────────────────────────
        404 │          0 │ 0.13333333333333333333 │ 0.34280333180088158345
        500 │          0 │ 0.15000000000000000000 │ 0.36008473579027553993
        200 │       4084 │  2779.1000000000000000 │       689.219644702665
        400 │         24 │ 0.73333333333333333333 │     3.4388935285299212

                  To get the last value in a GROUP BY in addition to the mean and standard deviation we used a little array trick.

                  Next, we calculate the z-score for the last value for each status code: 

                  db=# WITH stats AS (
   SELECT
       status_code,
       (MAX(ARRAY[EXTRACT('epoch' FROM period), entries]))[2] AS last_value,
       AVG(entries) AS mean_entries,
       STDDEV(entries) AS stddev_entries
   FROM
       server_log_summary
   WHERE
       -- In the demo data use:
       -- period > '2020-08-01 17:00 UTC'::timestamptz
       period > now() - interval '1 hour'
   GROUP BY
       status_code
)
SELECT
   *,
   (last_value - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
FROM
   stats;

status_code │ last_value │ mean_entries │ stddev_entries │  zscore
────────────┼────────────┼──────────────┼────────────────┼────────
        404 │          0 │ 0.133        │ 0.3428         │ -0.388
        500 │          0 │ 0.150        │ 0.3600         │ -0.416
        200 │       4084 │ 2779.100     │ 689.2196       │  1.893
        400 │         24 │ 0.733        │ 3.4388         │  6.765

                  We calculated the z-score by finding the number of standard deviations between the last value and the mean. To avoid a "division by zero" error we transform the denominator to NULL if it's zero.

                  Looking at the z-scores we got, we can spot that status code 400 got a very high z-score of 6. In the past minute we returned a 400 status code 24 times, which is significantly higher than the average of 0.73 in the past hour.

                  Let's take a look at the raw data: 

                  SELECT *
FROM server_log_summary
WHERE status_code = 400
ORDER BY period DESC
LIMIT 20;

        period         │ status_code │ entries
───────────────────────┼─────────────┼─────────
2020-08-01 18:00:00+00 │         400 │      24
2020-08-01 17:59:00+00 │         400 │      12
2020-08-01 17:58:00+00 │         400 │       2
2020-08-01 17:57:00+00 │         400 │       0
2020-08-01 17:56:00+00 │         400 │       1
2020-08-01 17:55:00+00 │         400 │       0
2020-08-01 17:54:00+00 │         400 │       0
2020-08-01 17:53:00+00 │         400 │       0
2020-08-01 17:52:00+00 │         400 │       0
2020-08-01 17:51:00+00 │         400 │       0
2020-08-01 17:50:00+00 │         400 │       0
2020-08-01 17:49:00+00 │         400 │       0
2020-08-01 17:48:00+00 │         400 │       0
2020-08-01 17:47:00+00 │         400 │       0
2020-08-01 17:46:00+00 │         400 │       0
2020-08-01 17:45:00+00 │         400 │       0
2020-08-01 17:44:00+00 │         400 │       0
2020-08-01 17:43:00+00 │         400 │       0
2020-08-01 17:42:00+00 │         400 │       0
2020-08-01 17:41:00+00 │         400 │       0

                  It does look like in the last couple of minutes we are getting more errors than expected.

                  Status 400 in the past hour
                  Status 400 in the past hour

                  What our naked eye missed in the chart and in the raw data, was found by the query, and was classified as an anomaly. We are off to a great start!

                  Backtesting

                  In the previous section we identified an anomaly. We found an increase in 400 status code because the z-score was 6. But how do we set the threshold for the z-score? Is a z-score of 3 an anomaly? What about 2, or 1?

                  To find thresholds that fit our needs, we can run simulations on past data with different values, and evaluate the results. This is often called backtesting.

                  Finding Past Anomalies

                  The first thing we need to do is to calculate the mean and the standard deviation for each status code up until every row, just as if it’s the current value. This is a classic job for a window function: 

                  WITH calculations_over_window AS (
   SELECT
      status_code,
      period,
      entries,
      AVG(entries) OVER status_window as mean_entries,
      STDDEV(entries) OVER status_window as stddev_entries
   FROM
      server_log_summary
   WINDOW status_window AS (
      PARTITION BY status_code
      ORDER BY period
      ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
)
SELECT *
FROM calculations_over_window
ORDER BY period DESC
LIMIT 20;

status_code │         period         │ entries │      mean_entries      │     stddev_entries
────────────┼────────────────────────┼─────────┼────────────────────────┼────────────────────────
        200 │ 2020-08-01 18:00:00+00 │    4084 │  2759.9672131147540984 │       699.597407256800
        400 │ 2020-08-01 18:00:00+00 │      24 │ 0.72131147540983606557 │     3.4114080550460080
        404 │ 2020-08-01 18:00:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        500 │ 2020-08-01 18:00:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735
        500 │ 2020-08-01 17:59:00+00 │       0 │ 0.16393442622950819672 │ 0.37328844382740000274
        400 │ 2020-08-01 17:59:00+00 │      12 │ 0.32786885245901639344 │     1.5676023249473471
        200 │ 2020-08-01 17:59:00+00 │    3927 │  2718.6721311475409836 │       694.466863171826
        404 │ 2020-08-01 17:59:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        500 │ 2020-08-01 17:58:00+00 │       0 │ 0.16393442622950819672 │ 0.37328844382740000274
        404 │ 2020-08-01 17:58:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        200 │ 2020-08-01 17:58:00+00 │    3850 │  2680.4754098360655738 │       690.967283512936
        400 │ 2020-08-01 17:58:00+00 │       2 │ 0.13114754098360655738 │ 0.38623869286861001780
        404 │ 2020-08-01 17:57:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        400 │ 2020-08-01 17:57:00+00 │       0 │ 0.09836065573770491803 │ 0.30027309973793774423
        500 │ 2020-08-01 17:57:00+00 │       1 │ 0.16393442622950819672 │ 0.37328844382740000274
        200 │ 2020-08-01 17:57:00+00 │    3702 │  2643.0327868852459016 │       688.414796645480
        200 │ 2020-08-01 17:56:00+00 │    3739 │  2607.5081967213114754 │       688.769908918569
        404 │ 2020-08-01 17:56:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735
        400 │ 2020-08-01 17:56:00+00 │       1 │ 0.11475409836065573770 │ 0.32137001808599097120
        500 │ 2020-08-01 17:56:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735

                  To calculate the mean and standard deviation over a sliding window of 60 minutes, we use a window function. To avoid having to repeat the WINDOW clause for every aggregate, we define a named window called "status_window". This is another nice feature of PostgreSQL.

                  In the results we can now see that for every entry, we have the mean and standard deviation of the previous 60 rows. This is similar to the calculation we did in the previous section, only this time we do it for every row.

                  Now we can calculate the z-score for every row: 

                  WITH calculations_over_window AS (
   SELECT
      status_code,
      period,
      entries,
      AVG(entries) OVER status_window as mean_entries,
      STDDEV(entries) OVER status_window as stddev_entries
   FROM
      server_log_summary
   WINDOW status_window AS (
      PARTITION BY status_code
      ORDER BY period
      ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
)

SELECT
   status_code,
   period,
   zscore
FROM
   with_zscore
ORDER BY
   period DESC
LIMIT
   20;

status_code │         period         │        zscore
────────────┼────────────────────────┼──────────────────────
        200 │ 2020-08-01 18:00:00+00 │   1.8925638848161648
        400 │ 2020-08-01 18:00:00+00 │    6.823777205473068
        404 │ 2020-08-01 18:00:00+00 │ -0.38531664163524526
        500 │ 2020-08-01 18:00:00+00 │ -0.41260101365496504
        500 │ 2020-08-01 17:59:00+00 │  -0.4391628750910588
        400 │ 2020-08-01 17:59:00+00 │    7.445849602151508
        200 │ 2020-08-01 17:59:00+00 │   1.7399359608515874
        404 │ 2020-08-01 17:59:00+00 │ -0.38531664163524526
        500 │ 2020-08-01 17:58:00+00 │  -0.4391628750910588
        404 │ 2020-08-01 17:58:00+00 │ -0.38531664163524526
        200 │ 2020-08-01 17:58:00+00 │   1.6925903990967166
        400 │ 2020-08-01 17:58:00+00 │    4.838594613958412
        404 │ 2020-08-01 17:57:00+00 │ -0.38531664163524526
        400 │ 2020-08-01 17:57:00+00 │ -0.32757065425956844
        500 │ 2020-08-01 17:57:00+00 │      2.2397306629644
        200 │ 2020-08-01 17:57:00+00 │   1.5382691050147506
        200 │ 2020-08-01 17:56:00+00 │   1.6427718293547886
        404 │ 2020-08-01 17:56:00+00 │ -0.41260101365496504
        400 │ 2020-08-01 17:56:00+00 │     2.75460015502278
        500 │ 2020-08-01 17:56:00+00 │ -0.41260101365496504

                  We now have z-scores for every row, and we can try to identify anomalies: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       zscore > 3 AS alert
   FROM
       with_zscore
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_alert
WHERE
   alert
ORDER BY
   period DESC
LIMIT
   20;

status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t
        400 │ 2020-08-01 17:58:00+00 │       2 │  4.838594613958412 │ t
        500 │ 2020-08-01 17:29:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 17:20:00+00 │       1 │ 3.3190952747131184 │ t
        500 │ 2020-08-01 17:18:00+00 │       1 │ 3.7438474117708043 │ t
        500 │ 2020-08-01 17:13:00+00 │       1 │ 3.7438474117708043 │ t
        500 │ 2020-08-01 17:09:00+00 │       1 │  4.360778994930029 │ t
        500 │ 2020-08-01 16:59:00+00 │       1 │ 3.7438474117708043 │ t
        400 │ 2020-08-01 16:29:00+00 │       1 │ 3.0027309973793774 │ t
        404 │ 2020-08-01 16:13:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 15:13:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 15:11:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 14:58:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:56:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:55:00+00 │       1 │ 3.3190952747131184 │ t
        400 │ 2020-08-01 14:50:00+00 │       1 │ 3.3190952747131184 │ t
        500 │ 2020-08-01 14:37:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:35:00+00 │       1 │ 3.3190952747131184 │ t
        400 │ 2020-08-01 14:32:00+00 │       1 │ 3.3190952747131184 │ t

                  We decided to classify values with z-score greater than 3 as anomalies. 3 is usually the magic number you’ll see in textbooks, but don’t get sentimental about it because you can definitely change it to get better results.

                  Adding Thresholds

                  In the last query we detected a large number of "anomalies" with just one entry. This is very common in errors that don't happen very often. In our case, every once in a while we get a 400 status code, but because it doesn't happen very often, the standard deviation is very low so that even a single error can be considered way above the acceptable value.

                  We don't really want to receive an alert in the middle of the night just because of one 400 status code. We can't have every curious developer fiddling with the devtools in his browser wake us up in the middle of the night.

                  To eliminate rows with only a few entries we set a threshold: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       entries > 10 AND zscore > 3 AS alert
   FROM
       with_zscore
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_alert
WHERE
   alert
ORDER BY
   period DESC;

status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t
        500 │ 2020-08-01 11:29:00+00 │    5001 │  3.172198441961645 │ t
        500 │ 2020-08-01 11:28:00+00 │    4812 │ 3.3971646910263917 │ t
        500 │ 2020-08-01 11:27:00+00 │    4443 │ 3.5349400089601586 │ t
        500 │ 2020-08-01 11:26:00+00 │    4522 │ 4.1264785335553595 │ t
        500 │ 2020-08-01 11:25:00+00 │    5567 │   6.17629336121081 │ t
        500 │ 2020-08-01 11:24:00+00 │    3657 │ 6.8689992361141154 │ t
        500 │ 2020-08-01 11:23:00+00 │    1512 │  6.342260662589681 │ t
        500 │ 2020-08-01 11:22:00+00 │    1022 │  7.682189672504754 │ t
        404 │ 2020-08-01 07:20:00+00 │      23 │  5.142126410098476 │ t
        404 │ 2020-08-01 07:19:00+00 │      20 │  6.091200697920824 │ t
        404 │ 2020-08-01 07:18:00+00 │      15 │   7.57547172423804 │ t

                  After eliminating potential anomalies with less than 10 entries we get much fewer, and probably more relevant results.

                  Eliminating Repeating Alerts

                  In the previous section we eliminated potential anomalies with less than 10 entries. Using thresholds we were able to remove some non interesting anomalies.

                  Let's have a look at the data for status code 400 after applying the threshold: 

                  status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t

                  The first alert happened in 17:59, and a minute later the z-score was still high with a large number of entries and so we classified the next rows at 18:00 as an anomaly as well.

                  If you think of an alerting system, we want to send an alert only when an anomaly first happens. We don't want to send an alert every minute until the z-score comes back below the threshold. In this case, we only want to send one alert at 17:59. We don't want to send another alert a minute later at 18:00.

                  Let's remove alerts where the previous period was also classified as an alert: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       entries > 10 AND zscore > 3 AS alert
   FROM
       with_zscore
),

with_previous_alert AS (
   SELECT
       *,
       LAG(alert) OVER (PARTITION BY status_code ORDER BY period) AS previous_alert
   FROM
       with_alert
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_previous_alert
WHERE
   alert AND NOT previous_alert
ORDER BY
   period DESC;

status_code │         period         │ entries │      zscore       │ alert
────────────┼────────────────────────┼─────────┼───────────────────┼───────
        400 │ 2020-08-01 17:59:00+00 │      12 │ 7.445849602151508 │ t
        500 │ 2020-08-01 11:22:00+00 │    1022 │ 7.682189672504754 │ t
        404 │ 2020-08-01 07:18:00+00 │      15 │  7.57547172423804 │ t

                  By eliminating alerts that were already triggered we get a very small list of anomalies that may have happened during the day. Looking at the results we can see what anomalies we would have discovered:

                  • Anomaly in status code 400 at 17:59: we also found that one earlier.
                  Anomaly in status code 400
                  Anomaly in status code 400
                  • Anomaly in status code 500: we spotted this one on the chart when we started.
                  Anomaly in status code 500
                  Anomaly in status code 500
                  • Anomaly in status code 404: this is a hidden hidden anomaly which we did not know about until now.
                  A hidden anomaly in status code 404
                  A hidden anomaly in status code 404

                  The query can now be used to fire alerts when it encounters an anomaly.

                  Experiment With Different Values

                  In the process so far we’ve used several constants in our calculations:

                  • Lookback period: How far back we calculate the mean and standard deviation for each status code. The value we used is 60 minutes.
                  • Entries Threshold: The least amount of entries we want to get an alert for. The value we used is 10.
                  • Z-Score Threshold: The z-score after which we classify the value as an anomaly. The value we used is 6.

                  Now that we have a working query to backtest, we can experiment with different values.

                  Experimenting with parameter values
                  Experimenting with parameter values

                  This is a chart showing the alerts our system identified in the past 12 hours:

                  Backtesting with default parameters. <a href="https://popsql.com/queries/-MECQV6GiKr04WdCWM0K/simple-anomaly-detection-with-sql?access_token=2d2c0729f9a1cfa7b6a2dbb5b0adb45c">View in editor</a>
                  Backtesting with default parameters. View in editor

                  To get a sense of each parameter, let's adjust the values and see how it affects the number and quality of alerts we get.

                  If we decrease the value of the z-score threshold from 3 to 1, we should get more alerts. With a lower threshold, more values are likely to be considered an anomaly:

                  Backtesting with lower z-score threshold
                  Backtesting with lower z-score threshold

                  If we increase the entries threshold from 10 to 30, we should get less alerts:

                  Backtesting with higher entries threshold
                  Backtesting with higher entries threshold

                  If we increase the backtest period from 60 minutes to 360 minutes, we get more alerts:

                  Backtesting with higher entries threshold
                  Backtesting with higher entries threshold

                  A good alerting system is a system that produces true alerts, at a reasonable time. Using the backtesting query you can experiment with different values that produces quality alerts you can act on.

                  Improving Accuracy

                  Using a z-score for detecting anomalies is an easy way to get started with anomaly detection and see results right away. But, this method is not always the best choice, and if you don't get good alerts using this method, there are some improvements and other methods you can try using just SQL.

                  Use Weighted Mean

                  Our system uses a mean to determine a reasonable value, and a lookback period to determine how long back to calculate that mean over. In our case, we calculated the mean based on data from 1 hour ago.

                  Using this method of calculating mean gives the same weight to entries that happened 1 hour ago and to entries that just happened. If you give more weight to recent entries at the expense of previous entries, the new weighted mean should become more sensitive to recent entries, and you may be able to identify anomalies quicker.

                  To give more weight to recent entries, you can use a weighted average: 

                  SELECT
   status_code,
   avg(entries) as mean,
   sum(
      entries *
      (60 - extract('seconds' from '2020-08-01 17:00 UTC'::timestamptz - period))
   ) / (60 * 61 / 2) as weighted_mean
FROM
   server_log_summary
WHERE
   -- Last 60 periods
   period > '2020-08-01 17:00 UTC'::timestamptz
GROUP BY
   status_code;

 status_code │          mean          │    weighted_mean
─────────────┼────────────────────────┼─────────────────────
         404 │ 0.13333333333333333333 │ 0.26229508196721313
         500 │ 0.15000000000000000000 │ 0.29508196721311475
         200 │  2779.1000000000000000 │   5467.081967213115
         400 │ 0.73333333333333333333 │  1.4426229508196722

                  In the results you can see the difference between the mean and the weighted mean for each status code.

                  A weighted average is a very common indicator used by stock traders. We used a linear weighted average, but there are also exponential weighted averages and others you can try.

                  Use Median

                  In statistics, a mean is considered not robust because it is influenced by extreme values. Given our use case, the measure we are using to identify extreme values, is affected by those values we are trying to identify.

                  For example, in the beginning of the article we used this series of values: 

                  2, 3, 5, 2, 3, 12, 5, 3, 4

                  The mean of this series is 4.33, and we detected 12 as an anomaly.

                  If the 12 were a 120, the mean of the series would have been 16.33. Hence, our "reasonable" value is heavily affected by the values it is supposed to identify.

                  A measure that is considered more robust is a median. The median of a series is the value that half the series is greater than, and half the series is less than: 

                  SELECT percentile_disc(0.5) within group(order by n)
FROM unnest(ARRAY[2, 3, 5, 2, 3, 120, 5, 3, 4]) as n;

 median
────────
      3

                  To calculate the median in PostgreSQL we use the function percentile_disc. In the series above, the median is 3. If we sort the list and cut it in the middle it will become more clear: 

                  2, 2, 3, 3, 3
4, 5, 5, 12

                  If we change the value of 12 to 120, the median will not be affected at all: 

                  2, 2, 3, 3, 3
4, 5, 5, 120

                  This is why a median is considered more robust than mean.

                  Use MAD

                  Median absolute deviation (MAD) is another way of finding anomalies in a series. MAD is considered better than z-score for real life data.

                  MAD is calculated by finding the median of the deviations from the series median. Just for comparison, the standard deviation is the root square of the average square distance from the mean.

                  Use Different Measures

                  We used the number of entries per minute as an indicator. However, depending on the use case, there might be other things you can measure that can yield better results. For example:

                  • To try and identify DOS attacks you can monitor the ratio between unique IP addresses to HTTP requests.
                  • To reduce the amount of false positives, you can normalize the number of responses to the proportion of the total responses. This way, for example, if you're using a flaky remote service that fails once after every certain amount of requests, using the proportion may not trigger an alert when the increase in errors correlates with an increase in overall traffic.

                  Conclusion

                  The method presented above is a very simple method to detect anomalies and produce actionable alerts that can potentially save you a lot of grief. There are many tools out there that provide similar functionally, but they require either tight integration or $$$. The main appeal of this approach is that you can get started with tools you probably already have, some SQL and a scheduled task!

                  UPDATE: many readers asked me how I created the charts in this article... well, I used PopSQL. It’s a new modern SQL editor focused on collaborative editing. If you're in the market for one, go check it out...

                  ================================================ FILE: test/test-pages/toc-missing/source.html ================================================ Simple Anomaly Detection Using Plain SQL | Haki Benita
                  • SQL

                  Simple Anomaly Detection Using Plain SQL

                  Identify Problems Before They Become Disasters

                  Many developers think that having a critical bug in their code is the worst thing that can happen. Well, there is something much worse than that: Having a critical bug in your code and not knowing about it!

                  To make sure I get notified about critical bugs as soon as possible, I started looking for ways to find anomalies in my data. I quickly found that information about these subjects tend to get very complicated, and involve a lot of ad-hoc tools and dependencies.

                  I'm not a statistician and not a data scientist, I'm just a developer. Before I introduce dependencies into my system I make sure I really can't do without them. So, using some high school level statistics and a fair knowledge of SQL, I implemented a simple anomaly detection system that works.

                  Can you spot the anomaly?<br><small>Photo by <a href="https://unsplash.com/photos/KmKZV8pso-s">Ricardo Gomez Angel</a></small>
                  Can you spot the anomaly?
                  Photo by Ricardo Gomez Angel
                  Table of Contents

                  Interactive Editor

                  To follow along with the article and experiment with actual data online check out the interactive editor on PopSQL ≫

                  Detecting Anomalies

                  Anomaly in a data series is a significant deviation from some reasonable value. Looking at this series of numbers for example, which number stands out? 

                  2, 3, 5, 2, 3, 12, 5, 3, 4

                  The number that stands out in this series is 12.

                  Scatter plot
                  Scatter plot

                  This is intuitive to a human, but computer programs don't have intuition...

                  To find the anomaly in the series we first need to define what a reasonable value is, and then define how far away from this value we consider a significant deviation. A good place to start looking for a reasonable value is the mean: 

                  SELECT avg(n)
FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

       avg
────────────────────
4.3333333333333333

                  The mean is ~4.33.

                  Next, we need to define the deviation. Let's use Standard Deviation: 

                  SELECT stddev(n)
FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

      stddev
────────────────────
3.0822070014844882

                  Standard deviation is the square root of the variance, which is the average squared distance from the mean. In this case it's 3.08.

                  Now that we've defined a "reasonable" value and a deviation, we can define a range of acceptable values: 

                  SELECT
   avg(n) - stddev(n) AS lower_bound,
   avg(n) + stddev(n) AS upper_bound
FROM
   unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n;

    lower_bound    │     upper_bound
───────────────────┼────────────────────
1.2511263318488451 │ 7.4155403348178215

                  The range we defined is one standard deviation from the mean. Any value outside this range is considered an anomaly: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
bounds AS (
   SELECT
       avg(n) - stddev(n) AS lower_bound,
       avg(n) + stddev(n) AS upper_bound
   FROM
       series
)
SELECT
   n,
   n NOT BETWEEN lower_bound AND upper_bound AS is_anomaly
FROM
   series,
   bounds;

n  │ is_anomaly
───┼────────────
 2 │ f
 3 │ f
 5 │ f
 2 │ f
 3 │ f
12 │ t
 5 │ f
 3 │ f
 4 │ f

                  Using the query we found that the value 12 is outside the range of acceptable values, and identified it as an anomaly.

                  Understanding Z-Score

                  Another way to represent a range of acceptable values is using a z-score. z-score, or Standard Score, is the number of standard deviations from the mean. In the previous section, our acceptable range was one standard deviation from the mean, or in other words, a z-score in the range ±1: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_mean,
       stddev(n) as series_stddev
   FROM
       series
)
SELECT
   n,
   (n - series_mean) / series_stddev as zscore
FROM
   series,
   stats;

n  │         zscore
───┼─────────────────────────
 2 │ -0.75703329861022517346
 3 │ -0.43259045634870009448
 5 │  0.21629522817435006346
 2 │ -0.75703329861022517346
 3 │ -0.43259045634870009448
12 │      2.4873951240050256
 5 │  0.21629522817435006346
 3 │ -0.43259045634870009448
 4 │ -0.10814761408717501551

                  Like before, we can detect anomalies by searching for values which are outside the acceptable range using the z-score: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_avg,
       stddev(n) as series_stddev
   FROM
       series
),
zscores AS (
   SELECT
       n,
       (n - series_avg) / series_stddev AS zscore
   FROM
       series,
       stats
)
SELECT
   *,
   zscore NOT BETWEEN -1 AND 1 AS is_anomaly
FROM
   zscores;

n  │         zscore          │ is_anomaly
───┼─────────────────────────┼────────────
 2 │ -0.75703329861022517346 │ f
 3 │ -0.43259045634870009448 │ f
 5 │  0.21629522817435006346 │ f
 2 │ -0.75703329861022517346 │ f
 3 │ -0.43259045634870009448 │ f
12 │      2.4873951240050256 │ t
 5 │  0.21629522817435006346 │ f
 3 │ -0.43259045634870009448 │ f
 4 │ -0.10814761408717501551 │ f

                  Using z-score, we also identified 12 as an anomaly in this series.

                  Optimizing Z-Score

                  So far we used one standard deviation from the mean, or a z-score of ±1 to identify anomalies. Changing the z-score threshold can affect our results. For example, let's see what anomalies we identify when the z-score is greater than 0.5 and when it's greater than 3: 

                  WITH series AS (
   SELECT *
   FROM unnest(array[2, 3, 5, 2, 3, 12, 5, 3, 4]) AS n
),
stats AS (
   SELECT
       avg(n) series_avg,
       stddev(n) as series_stddev
   FROM
       series
),
zscores AS (
   SELECT
       n,
       (n - series_avg) / series_stddev AS zscore
   FROM
       series,
       stats
)
SELECT
   *,
   zscore NOT BETWEEN -0.5 AND 0.5 AS is_anomaly_0_5,
   zscore NOT BETWEEN -1 AND 1 AS is_anomaly_1,
   zscore NOT BETWEEN -3 AND 3 AS is_anomaly_3
FROM
   zscores;

n  │         zscore          │ is_anomaly_0_5 │ is_anomaly_1 │ is_anomaly_3
───┼─────────────────────────┼────────────────┼──────────────┼──────────────
 2 │ -0.75703329861022517346 │ t              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
 5 │  0.21629522817435006346 │ f              │ f            │ f
 2 │ -0.75703329861022517346 │ t              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
12 │      2.4873951240050256 │ t              │ t            │ f
 5 │  0.21629522817435006346 │ f              │ f            │ f
 3 │ -0.43259045634870009448 │ f              │ f            │ f
 4 │ -0.10814761408717501551 │ f              │ f            │ f

                  Let's see what we got:

                  • When we decreased the z-score threshold to 0.5, we identified the value 2 as an anomaly in addition to the value 12.
                  • When we increased the z-score threshold to 3 we did not identify any anomaly.

                  The quality of our results are directly related to the parameters we set for the query. Later we'll see how using backtesting can help us identify ideal values.

                  Analyzing a Server Log

                  Application servers such as nginx, Apache and IIS write a lot of useful information to access logs. The data in these logs can be extremely useful in identifying anomalies.

                  We are going to analyze logs of a web application, so the data we are most interested in is the timestamp and the status code of every response from the server. To illustrate the type of insight we can draw from just this data:

                  • A sudden increase in 500 status code: You may have a problem in the server. Did you just push a new version? Is there an external service you're using that started failing in unexpected ways?
                  • A sudden increase in 400 status code: You may have a problem in the client. Did you change some validation logic and forgot to update the client? Did you make a change and forgot to handle backward compatibility?
                  • A sudden increase in 404 status code: You may have an SEO problem. Did you move some pages and forgot to set up redirects? Is there some script kiddy running a scan on your site?
                  • A sudden increase in 200 status code: You either have some significant legit traffic coming in, or you are under a DOS attack. Either way, you probably want to check where it's coming from.

                  Preparing the Data

                  Parsing and processing logs is outside the scope of this article, so let's assume we did that and we have a table that looks like this: 

                  CREATE TABLE server_log_summary AS (
   period timestamptz,
   status_code int,
   entries int
);

                  The table stores the number of entries for each status code at a given period. For example, our table stores how many responses returned each status code every minute: 

                  db=# SELECT * FROM server_log_summary ORDER BY period DESC LIMIT 10;

        period         │ status_code │ entries
───────────────────────┼─────────────┼─────────
2020-08-01 18:00:00+00 │         200 │    4084
2020-08-01 18:00:00+00 │         404 │       0
2020-08-01 18:00:00+00 │         400 │      24
2020-08-01 18:00:00+00 │         500 │       0
2020-08-01 17:59:00+00 │         400 │      12
2020-08-01 17:59:00+00 │         200 │    3927
2020-08-01 17:59:00+00 │         500 │       0
2020-08-01 17:59:00+00 │         404 │       0
2020-08-01 17:58:00+00 │         400 │       2
2020-08-01 17:58:00+00 │         200 │    3850

                  Note that the table has a row for every minute, even if the status code was never returned in that minute. Given a table of statuses, it's very tempting to do something like this: 

                  -- Wrong!
SELECT
   date_trunc('minute', timestamp) AS period,
   status_code,
   count(*) AS entries
FROM
   server_log
GROUP BY
   period,
   status_code;

                  This is a common mistake and it can leave you with gaps in the data. Zero is a value, and it holds a significant meaning. A better approach is to create an "axis", and join to it: 

                  -- Correct!
WITH axis AS (
   SELECT
       status_code,
       generate_series(
           date_trunc('minute', now()),
           date_trunc('minute', now() - interval '1 hour'),
           interval '1 minute' * -1
       ) AS period
   FROM (
       VALUES (200), (400), (404), (500)
   ) AS t(status_code)
)
SELECT
   a.period,
   a.status_code,
   count(*) AS entries
FROM
   axis a
   LEFT JOIN server_log l ON (
       date_trunc('minute', l.timestamp) = a.period
       AND l.status_code = a.status_code
   )
GROUP BY
   period,
   status_code;

                  First we generate an axis using a cartesian join between the status codes we want to track, and the times we want to monitor. To generate the axis we used two nice features of PostgreSQL:

                  • generate_series: function that generates a range of values.
                  • VALUES list: special clause that can generate "constant tables", as the documentation calls it. You might be familiar with the VALUES clause from INSERT statements. In the old days, to generate data we had to use a bunch of SELECT ... UNION ALL... using VALUES is much nicer.

                  After generating the axis, we left join the actual data into it to get a complete series for each status code. The resulting data has no gaps, and is ready for analysis.

                  Getting a Sense of the Data

                  To get a sense of the data, let's draw a stacked bar chart by status:

                  stacked bar chart by status, over time
                  stacked bar chart by status, over time

                  The chart shows a period of 12 hours. It looks like we have a nice trend with two peaks at around 09:30 and again at 18:00.

                  We also spot right away that at ~11:30 there was a significant increase in 500 errors. The burst died down after around 10 minutes. This is the type of anomalies we want to identify early on.

                  It's entirely possible that there were other problems during that time, we just can't spot them with a naked eye.

                  Identifying Anomalies

                  In anomaly detection systems, we usually want to identify if we have an anomaly right now, and send an alert.

                  To identify if the last datapoint is an anomaly, we start by calculating the mean and standard deviation for each status code in the past hour: 

                  db=# WITH stats AS (
   SELECT
       status_code,
       (MAX(ARRAY[EXTRACT('epoch' FROM period), entries]))[2] AS last_value,
       AVG(entries) AS mean_entries,
       STDDEV(entries) AS stddev_entries
   FROM
       server_log_summary
   WHERE
       -- In the demo data use:
       -- period > '2020-08-01 17:00 UTC'::timestamptz
       period > now() - interval '1 hour'
   GROUP BY
       status_code
)
SELECT * FROM stats;

status_code │ last_value │      mean_entries      │     stddev_entries
────────────┼────────────┼────────────────────────┼────────────────────────
        404 │          0 │ 0.13333333333333333333 │ 0.34280333180088158345
        500 │          0 │ 0.15000000000000000000 │ 0.36008473579027553993
        200 │       4084 │  2779.1000000000000000 │       689.219644702665
        400 │         24 │ 0.73333333333333333333 │     3.4388935285299212

                  To get the last value in a GROUP BY in addition to the mean and standard deviation we used a little array trick.

                  Next, we calculate the z-score for the last value for each status code: 

                  db=# WITH stats AS (
   SELECT
       status_code,
       (MAX(ARRAY[EXTRACT('epoch' FROM period), entries]))[2] AS last_value,
       AVG(entries) AS mean_entries,
       STDDEV(entries) AS stddev_entries
   FROM
       server_log_summary
   WHERE
       -- In the demo data use:
       -- period > '2020-08-01 17:00 UTC'::timestamptz
       period > now() - interval '1 hour'
   GROUP BY
       status_code
)
SELECT
   *,
   (last_value - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
FROM
   stats;

status_code │ last_value │ mean_entries │ stddev_entries │  zscore
────────────┼────────────┼──────────────┼────────────────┼────────
        404 │          0 │ 0.133        │ 0.3428         │ -0.388
        500 │          0 │ 0.150        │ 0.3600         │ -0.416
        200 │       4084 │ 2779.100     │ 689.2196       │  1.893
        400 │         24 │ 0.733        │ 3.4388         │  6.765

                  We calculated the z-score by finding the number of standard deviations between the last value and the mean. To avoid a "division by zero" error we transform the denominator to NULL if it's zero.

                  Looking at the z-scores we got, we can spot that status code 400 got a very high z-score of 6. In the past minute we returned a 400 status code 24 times, which is significantly higher than the average of 0.73 in the past hour.

                  Let's take a look at the raw data: 

                  SELECT *
FROM server_log_summary
WHERE status_code = 400
ORDER BY period DESC
LIMIT 20;

        period         │ status_code │ entries
───────────────────────┼─────────────┼─────────
2020-08-01 18:00:00+00 │         400 │      24
2020-08-01 17:59:00+00 │         400 │      12
2020-08-01 17:58:00+00 │         400 │       2
2020-08-01 17:57:00+00 │         400 │       0
2020-08-01 17:56:00+00 │         400 │       1
2020-08-01 17:55:00+00 │         400 │       0
2020-08-01 17:54:00+00 │         400 │       0
2020-08-01 17:53:00+00 │         400 │       0
2020-08-01 17:52:00+00 │         400 │       0
2020-08-01 17:51:00+00 │         400 │       0
2020-08-01 17:50:00+00 │         400 │       0
2020-08-01 17:49:00+00 │         400 │       0
2020-08-01 17:48:00+00 │         400 │       0
2020-08-01 17:47:00+00 │         400 │       0
2020-08-01 17:46:00+00 │         400 │       0
2020-08-01 17:45:00+00 │         400 │       0
2020-08-01 17:44:00+00 │         400 │       0
2020-08-01 17:43:00+00 │         400 │       0
2020-08-01 17:42:00+00 │         400 │       0
2020-08-01 17:41:00+00 │         400 │       0

                  It does look like in the last couple of minutes we are getting more errors than expected.

                  Status 400 in the past hour
                  Status 400 in the past hour

                  What our naked eye missed in the chart and in the raw data, was found by the query, and was classified as an anomaly. We are off to a great start!

                  Backtesting

                  In the previous section we identified an anomaly. We found an increase in 400 status code because the z-score was 6. But how do we set the threshold for the z-score? Is a z-score of 3 an anomaly? What about 2, or 1?

                  To find thresholds that fit our needs, we can run simulations on past data with different values, and evaluate the results. This is often called backtesting.

                  Finding Past Anomalies

                  The first thing we need to do is to calculate the mean and the standard deviation for each status code up until every row, just as if it’s the current value. This is a classic job for a window function: 

                  WITH calculations_over_window AS (
   SELECT
      status_code,
      period,
      entries,
      AVG(entries) OVER status_window as mean_entries,
      STDDEV(entries) OVER status_window as stddev_entries
   FROM
      server_log_summary
   WINDOW status_window AS (
      PARTITION BY status_code
      ORDER BY period
      ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
)
SELECT *
FROM calculations_over_window
ORDER BY period DESC
LIMIT 20;

status_code │         period         │ entries │      mean_entries      │     stddev_entries
────────────┼────────────────────────┼─────────┼────────────────────────┼────────────────────────
        200 │ 2020-08-01 18:00:00+00 │    4084 │  2759.9672131147540984 │       699.597407256800
        400 │ 2020-08-01 18:00:00+00 │      24 │ 0.72131147540983606557 │     3.4114080550460080
        404 │ 2020-08-01 18:00:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        500 │ 2020-08-01 18:00:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735
        500 │ 2020-08-01 17:59:00+00 │       0 │ 0.16393442622950819672 │ 0.37328844382740000274
        400 │ 2020-08-01 17:59:00+00 │      12 │ 0.32786885245901639344 │     1.5676023249473471
        200 │ 2020-08-01 17:59:00+00 │    3927 │  2718.6721311475409836 │       694.466863171826
        404 │ 2020-08-01 17:59:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        500 │ 2020-08-01 17:58:00+00 │       0 │ 0.16393442622950819672 │ 0.37328844382740000274
        404 │ 2020-08-01 17:58:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        200 │ 2020-08-01 17:58:00+00 │    3850 │  2680.4754098360655738 │       690.967283512936
        400 │ 2020-08-01 17:58:00+00 │       2 │ 0.13114754098360655738 │ 0.38623869286861001780
        404 │ 2020-08-01 17:57:00+00 │       0 │ 0.13114754098360655738 │ 0.34036303344446665347
        400 │ 2020-08-01 17:57:00+00 │       0 │ 0.09836065573770491803 │ 0.30027309973793774423
        500 │ 2020-08-01 17:57:00+00 │       1 │ 0.16393442622950819672 │ 0.37328844382740000274
        200 │ 2020-08-01 17:57:00+00 │    3702 │  2643.0327868852459016 │       688.414796645480
        200 │ 2020-08-01 17:56:00+00 │    3739 │  2607.5081967213114754 │       688.769908918569
        404 │ 2020-08-01 17:56:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735
        400 │ 2020-08-01 17:56:00+00 │       1 │ 0.11475409836065573770 │ 0.32137001808599097120
        500 │ 2020-08-01 17:56:00+00 │       0 │ 0.14754098360655737705 │ 0.35758754516763638735

                  To calculate the mean and standard deviation over a sliding window of 60 minutes, we use a window function. To avoid having to repeat the WINDOW clause for every aggregate, we define a named window called "status_window". This is another nice feature of PostgreSQL.

                  In the results we can now see that for every entry, we have the mean and standard deviation of the previous 60 rows. This is similar to the calculation we did in the previous section, only this time we do it for every row.

                  Now we can calculate the z-score for every row: 

                  WITH calculations_over_window AS (
   SELECT
      status_code,
      period,
      entries,
      AVG(entries) OVER status_window as mean_entries,
      STDDEV(entries) OVER status_window as stddev_entries
   FROM
      server_log_summary
   WINDOW status_window AS (
      PARTITION BY status_code
      ORDER BY period
      ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
)

SELECT
   status_code,
   period,
   zscore
FROM
   with_zscore
ORDER BY
   period DESC
LIMIT
   20;

status_code │         period         │        zscore
────────────┼────────────────────────┼──────────────────────
        200 │ 2020-08-01 18:00:00+00 │   1.8925638848161648
        400 │ 2020-08-01 18:00:00+00 │    6.823777205473068
        404 │ 2020-08-01 18:00:00+00 │ -0.38531664163524526
        500 │ 2020-08-01 18:00:00+00 │ -0.41260101365496504
        500 │ 2020-08-01 17:59:00+00 │  -0.4391628750910588
        400 │ 2020-08-01 17:59:00+00 │    7.445849602151508
        200 │ 2020-08-01 17:59:00+00 │   1.7399359608515874
        404 │ 2020-08-01 17:59:00+00 │ -0.38531664163524526
        500 │ 2020-08-01 17:58:00+00 │  -0.4391628750910588
        404 │ 2020-08-01 17:58:00+00 │ -0.38531664163524526
        200 │ 2020-08-01 17:58:00+00 │   1.6925903990967166
        400 │ 2020-08-01 17:58:00+00 │    4.838594613958412
        404 │ 2020-08-01 17:57:00+00 │ -0.38531664163524526
        400 │ 2020-08-01 17:57:00+00 │ -0.32757065425956844
        500 │ 2020-08-01 17:57:00+00 │      2.2397306629644
        200 │ 2020-08-01 17:57:00+00 │   1.5382691050147506
        200 │ 2020-08-01 17:56:00+00 │   1.6427718293547886
        404 │ 2020-08-01 17:56:00+00 │ -0.41260101365496504
        400 │ 2020-08-01 17:56:00+00 │     2.75460015502278
        500 │ 2020-08-01 17:56:00+00 │ -0.41260101365496504

                  We now have z-scores for every row, and we can try to identify anomalies: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       zscore > 3 AS alert
   FROM
       with_zscore
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_alert
WHERE
   alert
ORDER BY
   period DESC
LIMIT
   20;

status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t
        400 │ 2020-08-01 17:58:00+00 │       2 │  4.838594613958412 │ t
        500 │ 2020-08-01 17:29:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 17:20:00+00 │       1 │ 3.3190952747131184 │ t
        500 │ 2020-08-01 17:18:00+00 │       1 │ 3.7438474117708043 │ t
        500 │ 2020-08-01 17:13:00+00 │       1 │ 3.7438474117708043 │ t
        500 │ 2020-08-01 17:09:00+00 │       1 │  4.360778994930029 │ t
        500 │ 2020-08-01 16:59:00+00 │       1 │ 3.7438474117708043 │ t
        400 │ 2020-08-01 16:29:00+00 │       1 │ 3.0027309973793774 │ t
        404 │ 2020-08-01 16:13:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 15:13:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 15:11:00+00 │       1 │ 3.0027309973793774 │ t
        500 │ 2020-08-01 14:58:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:56:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:55:00+00 │       1 │ 3.3190952747131184 │ t
        400 │ 2020-08-01 14:50:00+00 │       1 │ 3.3190952747131184 │ t
        500 │ 2020-08-01 14:37:00+00 │       1 │ 3.0027309973793774 │ t
        400 │ 2020-08-01 14:35:00+00 │       1 │ 3.3190952747131184 │ t
        400 │ 2020-08-01 14:32:00+00 │       1 │ 3.3190952747131184 │ t

                  We decided to classify values with z-score greater than 3 as anomalies. 3 is usually the magic number you’ll see in textbooks, but don’t get sentimental about it because you can definitely change it to get better results.

                  Adding Thresholds

                  In the last query we detected a large number of "anomalies" with just one entry. This is very common in errors that don't happen very often. In our case, every once in a while we get a 400 status code, but because it doesn't happen very often, the standard deviation is very low so that even a single error can be considered way above the acceptable value.

                  We don't really want to receive an alert in the middle of the night just because of one 400 status code. We can't have every curious developer fiddling with the devtools in his browser wake us up in the middle of the night.

                  To eliminate rows with only a few entries we set a threshold: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       entries > 10 AND zscore > 3 AS alert
   FROM
       with_zscore
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_alert
WHERE
   alert
ORDER BY
   period DESC;

status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t
        500 │ 2020-08-01 11:29:00+00 │    5001 │  3.172198441961645 │ t
        500 │ 2020-08-01 11:28:00+00 │    4812 │ 3.3971646910263917 │ t
        500 │ 2020-08-01 11:27:00+00 │    4443 │ 3.5349400089601586 │ t
        500 │ 2020-08-01 11:26:00+00 │    4522 │ 4.1264785335553595 │ t
        500 │ 2020-08-01 11:25:00+00 │    5567 │   6.17629336121081 │ t
        500 │ 2020-08-01 11:24:00+00 │    3657 │ 6.8689992361141154 │ t
        500 │ 2020-08-01 11:23:00+00 │    1512 │  6.342260662589681 │ t
        500 │ 2020-08-01 11:22:00+00 │    1022 │  7.682189672504754 │ t
        404 │ 2020-08-01 07:20:00+00 │      23 │  5.142126410098476 │ t
        404 │ 2020-08-01 07:19:00+00 │      20 │  6.091200697920824 │ t
        404 │ 2020-08-01 07:18:00+00 │      15 │   7.57547172423804 │ t

                  After eliminating potential anomalies with less than 10 entries we get much fewer, and probably more relevant results.

                  Eliminating Repeating Alerts

                  In the previous section we eliminated potential anomalies with less than 10 entries. Using thresholds we were able to remove some non interesting anomalies.

                  Let's have a look at the data for status code 400 after applying the threshold: 

                  status_code │         period         │ entries │       zscore       │ alert
────────────┼────────────────────────┼─────────┼────────────────────┼───────
        400 │ 2020-08-01 18:00:00+00 │      24 │  6.823777205473068 │ t
        400 │ 2020-08-01 17:59:00+00 │      12 │  7.445849602151508 │ t

                  The first alert happened in 17:59, and a minute later the z-score was still high with a large number of entries and so we classified the next rows at 18:00 as an anomaly as well.

                  If you think of an alerting system, we want to send an alert only when an anomaly first happens. We don't want to send an alert every minute until the z-score comes back below the threshold. In this case, we only want to send one alert at 17:59. We don't want to send another alert a minute later at 18:00.

                  Let's remove alerts where the previous period was also classified as an alert: 

                  WITH calculations_over_window AS (
   SELECT
       status_code,
       period,
       entries,
       AVG(entries) OVER status_window as mean_entries,
       STDDEV(entries) OVER status_window as stddev_entries
   FROM
       server_log_summary
   WINDOW status_window AS (
       PARTITION BY status_code
       ORDER BY period
       ROWS BETWEEN 60 PRECEDING AND CURRENT ROW
   )
),

with_zscore AS (
   SELECT
       *,
       (entries - mean_entries) / NULLIF(stddev_entries::float, 0) as zscore
   FROM
       calculations_over_window
),

with_alert AS (

   SELECT
       *,
       entries > 10 AND zscore > 3 AS alert
   FROM
       with_zscore
),

with_previous_alert AS (
   SELECT
       *,
       LAG(alert) OVER (PARTITION BY status_code ORDER BY period) AS previous_alert
   FROM
       with_alert
)

SELECT
   status_code,
   period,
   entries,
   zscore,
   alert
FROM
   with_previous_alert
WHERE
   alert AND NOT previous_alert
ORDER BY
   period DESC;

status_code │         period         │ entries │      zscore       │ alert
────────────┼────────────────────────┼─────────┼───────────────────┼───────
        400 │ 2020-08-01 17:59:00+00 │      12 │ 7.445849602151508 │ t
        500 │ 2020-08-01 11:22:00+00 │    1022 │ 7.682189672504754 │ t
        404 │ 2020-08-01 07:18:00+00 │      15 │  7.57547172423804 │ t

                  By eliminating alerts that were already triggered we get a very small list of anomalies that may have happened during the day. Looking at the results we can see what anomalies we would have discovered:

                  • Anomaly in status code 400 at 17:59: we also found that one earlier.
                  Anomaly in status code 400
                  Anomaly in status code 400
                  • Anomaly in status code 500: we spotted this one on the chart when we started.
                  Anomaly in status code 500
                  Anomaly in status code 500
                  • Anomaly in status code 404: this is a hidden hidden anomaly which we did not know about until now.
                  A hidden anomaly in status code 404
                  A hidden anomaly in status code 404

                  The query can now be used to fire alerts when it encounters an anomaly.

                  Experiment With Different Values

                  In the process so far we’ve used several constants in our calculations:

                  • Lookback period: How far back we calculate the mean and standard deviation for each status code. The value we used is 60 minutes.
                  • Entries Threshold: The least amount of entries we want to get an alert for. The value we used is 10.
                  • Z-Score Threshold: The z-score after which we classify the value as an anomaly. The value we used is 6.

                  Now that we have a working query to backtest, we can experiment with different values.

                  Experimenting with parameter values
                  Experimenting with parameter values

                  This is a chart showing the alerts our system identified in the past 12 hours:

                  Backtesting with default parameters. <a href="https://popsql.com/queries/-MECQV6GiKr04WdCWM0K/simple-anomaly-detection-with-sql?access_token=2d2c0729f9a1cfa7b6a2dbb5b0adb45c">View in editor</a>
                  Backtesting with default parameters. View in editor

                  To get a sense of each parameter, let's adjust the values and see how it affects the number and quality of alerts we get.

                  If we decrease the value of the z-score threshold from 3 to 1, we should get more alerts. With a lower threshold, more values are likely to be considered an anomaly:

                  Backtesting with lower z-score threshold
                  Backtesting with lower z-score threshold

                  If we increase the entries threshold from 10 to 30, we should get less alerts:

                  Backtesting with higher entries threshold
                  Backtesting with higher entries threshold

                  If we increase the backtest period from 60 minutes to 360 minutes, we get more alerts:

                  Backtesting with higher entries threshold
                  Backtesting with higher entries threshold

                  A good alerting system is a system that produces true alerts, at a reasonable time. Using the backtesting query you can experiment with different values that produces quality alerts you can act on.

                  Improving Accuracy

                  Using a z-score for detecting anomalies is an easy way to get started with anomaly detection and see results right away. But, this method is not always the best choice, and if you don't get good alerts using this method, there are some improvements and other methods you can try using just SQL.

                  Use Weighted Mean

                  Our system uses a mean to determine a reasonable value, and a lookback period to determine how long back to calculate that mean over. In our case, we calculated the mean based on data from 1 hour ago.

                  Using this method of calculating mean gives the same weight to entries that happened 1 hour ago and to entries that just happened. If you give more weight to recent entries at the expense of previous entries, the new weighted mean should become more sensitive to recent entries, and you may be able to identify anomalies quicker.

                  To give more weight to recent entries, you can use a weighted average: 

                  SELECT
   status_code,
   avg(entries) as mean,
   sum(
      entries *
      (60 - extract('seconds' from '2020-08-01 17:00 UTC'::timestamptz - period))
   ) / (60 * 61 / 2) as weighted_mean
FROM
   server_log_summary
WHERE
   -- Last 60 periods
   period > '2020-08-01 17:00 UTC'::timestamptz
GROUP BY
   status_code;

 status_code │          mean          │    weighted_mean
─────────────┼────────────────────────┼─────────────────────
         404 │ 0.13333333333333333333 │ 0.26229508196721313
         500 │ 0.15000000000000000000 │ 0.29508196721311475
         200 │  2779.1000000000000000 │   5467.081967213115
         400 │ 0.73333333333333333333 │  1.4426229508196722

                  In the results you can see the difference between the mean and the weighted mean for each status code.

                  A weighted average is a very common indicator used by stock traders. We used a linear weighted average, but there are also exponential weighted averages and others you can try.

                  Use Median

                  In statistics, a mean is considered not robust because it is influenced by extreme values. Given our use case, the measure we are using to identify extreme values, is affected by those values we are trying to identify.

                  For example, in the beginning of the article we used this series of values: 

                  2, 3, 5, 2, 3, 12, 5, 3, 4

                  The mean of this series is 4.33, and we detected 12 as an anomaly.

                  If the 12 were a 120, the mean of the series would have been 16.33. Hence, our "reasonable" value is heavily affected by the values it is supposed to identify.

                  A measure that is considered more robust is a median. The median of a series is the value that half the series is greater than, and half the series is less than: 

                  SELECT percentile_disc(0.5) within group(order by n)
FROM unnest(ARRAY[2, 3, 5, 2, 3, 120, 5, 3, 4]) as n;

 median
────────
      3

                  To calculate the median in PostgreSQL we use the function percentile_disc. In the series above, the median is 3. If we sort the list and cut it in the middle it will become more clear: 

                  2, 2, 3, 3, 3
4, 5, 5, 12

                  If we change the value of 12 to 120, the median will not be affected at all: 

                  2, 2, 3, 3, 3
4, 5, 5, 120

                  This is why a median is considered more robust than mean.

                  Use MAD

                  Median absolute deviation (MAD) is another way of finding anomalies in a series. MAD is considered better than z-score for real life data.

                  MAD is calculated by finding the median of the deviations from the series median. Just for comparison, the standard deviation is the root square of the average square distance from the mean.

                  Use Different Measures

                  We used the number of entries per minute as an indicator. However, depending on the use case, there might be other things you can measure that can yield better results. For example:

                  • To try and identify DOS attacks you can monitor the ratio between unique IP addresses to HTTP requests.
                  • To reduce the amount of false positives, you can normalize the number of responses to the proportion of the total responses. This way, for example, if you're using a flaky remote service that fails once after every certain amount of requests, using the proportion may not trigger an alert when the increase in errors correlates with an increase in overall traffic.

                  Conclusion

                  The method presented above is a very simple method to detect anomalies and produce actionable alerts that can potentially save you a lot of grief. There are many tools out there that provide similar functionally, but they require either tight integration or $$$. The main appeal of this approach is that you can get started with tools you probably already have, some SQL and a scheduled task!

                  UPDATE: many readers asked me how I created the charts in this article... well, I used PopSQL. It’s a new modern SQL editor focused on collaborative editing. If you're in the market for one, go check it out...

                  Similar articles

                  ================================================ FILE: test/test-pages/topicseed-1/expected-metadata.json ================================================ { "title": "Content Depth — Write Comprehensively About Your Core Topics", "byline": null, "dir": null, "excerpt": "Content writers and marketers find it hard to write a lot of content about a very specific topic. They lose a lot of points on their content depth because they would rather focus on pushing thin content about plenty of topics.", "siteName": "topicseed", "publishedTime": "2018-06-12T23:00:00.000Z", "readerable": true } ================================================ FILE: test/test-pages/topicseed-1/expected.html ================================================

                  Content depth is an arbitrary score or rating of how comprehensive the coverage of a specific topic is within a piece of content. Content breadth is an arbitrary grading of how many related subjects are you covering within your content.

                  And this distinction is important to make and establish from the beginning. Effective topical authority can only be gained when you use both content depth and content breadth in your overall content strategy for rapid search engine optimization gains. However, because most content writers prefer to write a little bit about many things rather than write a lot about one thing, you end up with a too little substance spread very thin.

                  Content depth should be the urgent priority for your content marketing strategy, and clearly defined in your content briefs. Start by dominating your own core topics, before venturing across the pond and write about linked subject matters. Otherwise, you are the opposite of an authority as the definition states that an authority is “a person with extensive or specialized knowledge about a subject; an expert”. Lastly, do not mistake article depth vs. article length: a blog post’s extreme wordcount has nothing to do with its content depth.

                  Assess How Deep Is Your Content

                  The first task on your list, right now, is to shortlist your core topics. What are you trying to be an expert on? Then, go through each one of your pieces of content and understand how well each blog post is covering its focus topic(s). Not how many times specific keywords appear, or how well the article is outlined and structured.

                  Put yourself in the shoes of an ignorant reader who seeks information. Read your article. And ask yourself how in-depth was the content you have written? I know the excuse you will come up with: this was written for beginners, therefore, it shouldn’t be too in-depth. And you are correct. Not every blog post is about absolute content depth otherwise we would only write one 10,000-word-long article, once and for all. But then, how well your beginner-level content pointing to your expert-level content?

                  In other words, each article should reach an incredible level of content depth for its expertise level. And then, provide further reading (i.e. links) to gain more knowledge, and depth. A lot of content editors write a beginner’s blog post and wait to see it perform well in order to write a more advanced sequel. Wrong. Give all the value so search engines can grade you highly on their authority scale for your core topics. Yes, it is a risk and you may write a dozen of articles on a specific topic that will never really rank at the top of SERPs, but reaching content depth is the first step towards SEO gains.

                  Remember that skyscraper content and 10x content are not necessarily the answer. These content writing strategies state that in order to beat another piece of content, you need to write 10x more. Either in quantity with a 10x word count or in quality by putting times more information within your own piece of content. Such articles often become unreadable and discourage visitors from absorbing all the knowledge. The best alternative is the create pillar pages centered around core topics, and several articles dealing with each specific section in depth. This is deep content powered by a smart internal linking strategy and search engines love that in this day and age where attention spans are short! With that being said, avoid writing 600-word articles!

                  Rewrite With Content Depth In Mind

                  Once you know which articles are lacking depth of knowledge and information, it is time to rethink each one. For each article, make a list of what essential pieces of information or data are missing. Then decide where to fit them, and decide whether the article would benefit from a full rewrite or not. As a rule of thumb, if you need to change a third of your article, you may need to rewrite it entirely. Of course, this does not mean erasing all work done prior, but it means starting afresh! Trying to fit deep content into an existing blog post gives you constraints so doing it from scratch can actually be easier to fight thin content.

                  As explained above, make sure you do not force yourself to write a much longer article to reach a magic word count. And if you do, it has to be natural. In many cases, articles written months or years ago may need some upkeeping: trimming the fat and removing parts that are not bringing much value. Replace these with your newer and deeper content.

                  All content writers know that when you open Google Docs, WordPress, or your text editor of choice, you will inevitably count your focus keywords’ frequency. Although I understand (yet question) the value of keywords in modern SEO, do not become obsessed with reaching a magic number for your keywords. No reader coming from Google is out there counting how often your keywords are appearing. And search engine algorithms will penalize you for writing for robots, rather than humans.

                  With the massive rise of voice searches, users tend to use full questions for their search queries. What used to be top bottled water brands is now OK google, what is the best bottled-water brand in Texas? The point being, keywords are losing traction to leave space for a more natural language understanding of a blog post’s textual content, and meaning.

                  Yes, Content Depth and Breadth Overlap

                  “A topic can be defined as the company it keeps.” A very accurate saying loved by ontologists within the fields of computational linguistics, and information science. In simpler terms, a topic and all the terminology it is encompassing will inevitably overlap with related topics. Which, in turn, will form topic clusters.

                  For example, it is obvious that despite being two different topics, digital advertising and content marketing share some common phrases and terms. Inevitably, a website picking one as its core topic will use words in some blog posts that will identify the article as belonging to both topics, with a specific weight for each.

                  A keyword, phrase, or term, is not a prisoner to a single concept at all. This is how algorithms in natural language understanding can understand how two topics are related (e.g. read about topic modeling). Each topic has a specific vocabulary, a list of words and phrases commonly used in its context, and some of these terms are present in different vocabularies.

                  Therefore, content depth and content breadth are not to be opposed. Content marketers should use both strategies in order to reach ultimate topical authority over their choice of subject matters.

                  Depth of Content = Quality + Frequency

                  Up until recently, long-form blog posts generally were evergreen articles that generated a constant stream of organic traffic for a website. This was a lead magnet generation strategy which worked well: hire a writer, include the right keywords, reach over a 5,000-word word count, and hit publish. Then, wait.

                  Nowadays, in-depth content requires more effort over time in order to pay off. Writing a big article, as good as it is, will not get your anywhere near the level of topical breadth required by Google to rank you first. Instead, your content marketing plan should be about having:

                  • a comprehensive pillar page covering a unique topic, and
                  • narrow-focused children articles to dig deeper.

                  Search engines also look at how often you publish about a specific topic, and when was the last time it was written about. Nobody likes a graveyard blog, it just makes the reader lose trust; as if the writer was not good enough, therefore had no traffic, before entirely giving up. Deep content requires a sustained effort on your part to always new find ways to write about a specific subject. Sure, it will be easy at first. But what about five years later? Well, you will still need to hit publish, all about the very same topics you already covered years ago.

                  Tools and platforms such as topicseed are here to help you find new article ideas pertaining to your core topics within a few clicks and a few minutes. The number of web pages, Wikipedia articles, and pieces of content, our machine-learning algorithms can analyze in seconds would take you months to digest. Our topicgraph finds closely related concepts in order for your domain to reach topical authority through content depth and content breadth.

                  ================================================ FILE: test/test-pages/topicseed-1/source.html ================================================ Content Depth — Write Comprehensively About Your Core Topics | topicseed

                  Content Depth — Write Comprehensively About Your Core Topics

                  Content writers and marketers find it hard to write a lot of content about a very specific topic. They lose a lot of points on their content depth because they would rather focus on pushing thin content about plenty of topics.

                  • On 6/13/2018 by @lazharichir

                  Content depth is an arbitrary score or rating of how comprehensive the coverage of a specific topic is within a piece of content. Content breadth is an arbitrary grading of how many related subjects are you covering within your content.

                  And this distinction is important to make and establish from the beginning. Effective topical authority can only be gained when you use both content depth and content breadth in your overall content strategy for rapid search engine optimization gains. However, because most content writers prefer to write a little bit about many things rather than write a lot about one thing, you end up with a too little substance spread very thin.

                  Content depth should be the urgent priority for your content marketing strategy, and clearly defined in your content briefs. Start by dominating your own core topics, before venturing across the pond and write about linked subject matters. Otherwise, you are the opposite of an authority as the definition states that an authority is “a person with extensive or specialized knowledge about a subject; an expert”. Lastly, do not mistake article depth vs. article length: a blog post’s extreme wordcount has nothing to do with its content depth.

                  Assess How Deep Is Your Content

                  The first task on your list, right now, is to shortlist your core topics. What are you trying to be an expert on? Then, go through each one of your pieces of content and understand how well each blog post is covering its focus topic(s). Not how many times specific keywords appear, or how well the article is outlined and structured.

                  Put yourself in the shoes of an ignorant reader who seeks information. Read your article. And ask yourself how in-depth was the content you have written? I know the excuse you will come up with: this was written for beginners, therefore, it shouldn’t be too in-depth. And you are correct. Not every blog post is about absolute content depth otherwise we would only write one 10,000-word-long article, once and for all. But then, how well your beginner-level content pointing to your expert-level content?

                  In other words, each article should reach an incredible level of content depth for its expertise level. And then, provide further reading (i.e. links) to gain more knowledge, and depth. A lot of content editors write a beginner’s blog post and wait to see it perform well in order to write a more advanced sequel. Wrong. Give all the value so search engines can grade you highly on their authority scale for your core topics. Yes, it is a risk and you may write a dozen of articles on a specific topic that will never really rank at the top of SERPs, but reaching content depth is the first step towards SEO gains.

                  Remember that skyscraper content and 10x content are not necessarily the answer. These content writing strategies state that in order to beat another piece of content, you need to write 10x more. Either in quantity with a 10x word count or in quality by putting times more information within your own piece of content. Such articles often become unreadable and discourage visitors from absorbing all the knowledge. The best alternative is the create pillar pages centered around core topics, and several articles dealing with each specific section in depth. This is deep content powered by a smart internal linking strategy and search engines love that in this day and age where attention spans are short! With that being said, avoid writing 600-word articles!

                  Rewrite With Content Depth In Mind

                  Once you know which articles are lacking depth of knowledge and information, it is time to rethink each one. For each article, make a list of what essential pieces of information or data are missing. Then decide where to fit them, and decide whether the article would benefit from a full rewrite or not. As a rule of thumb, if you need to change a third of your article, you may need to rewrite it entirely. Of course, this does not mean erasing all work done prior, but it means starting afresh! Trying to fit deep content into an existing blog post gives you constraints so doing it from scratch can actually be easier to fight thin content.

                  As explained above, make sure you do not force yourself to write a much longer article to reach a magic word count. And if you do, it has to be natural. In many cases, articles written months or years ago may need some upkeeping: trimming the fat and removing parts that are not bringing much value. Replace these with your newer and deeper content.

                  All content writers know that when you open Google Docs, WordPress, or your text editor of choice, you will inevitably count your focus keywords’ frequency. Although I understand (yet question) the value of keywords in modern SEO, do not become obsessed with reaching a magic number for your keywords. No reader coming from Google is out there counting how often your keywords are appearing. And search engine algorithms will penalize you for writing for robots, rather than humans.

                  With the massive rise of voice searches, users tend to use full questions for their search queries. What used to be top bottled water brands is now OK google, what is the best bottled-water brand in Texas? The point being, keywords are losing traction to leave space for a more natural language understanding of a blog post’s textual content, and meaning.

                  Yes, Content Depth and Breadth Overlap

                  “A topic can be defined as the company it keeps.” A very accurate saying loved by ontologists within the fields of computational linguistics, and information science. In simpler terms, a topic and all the terminology it is encompassing will inevitably overlap with related topics. Which, in turn, will form topic clusters.

                  For example, it is obvious that despite being two different topics, digital advertising and content marketing share some common phrases and terms. Inevitably, a website picking one as its core topic will use words in some blog posts that will identify the article as belonging to both topics, with a specific weight for each.

                  A keyword, phrase, or term, is not a prisoner to a single concept at all. This is how algorithms in natural language understanding can understand how two topics are related (e.g. read about topic modeling). Each topic has a specific vocabulary, a list of words and phrases commonly used in its context, and some of these terms are present in different vocabularies.

                  Therefore, content depth and content breadth are not to be opposed. Content marketers should use both strategies in order to reach ultimate topical authority over their choice of subject matters.

                  Depth of Content = Quality + Frequency

                  Up until recently, long-form blog posts generally were evergreen articles that generated a constant stream of organic traffic for a website. This was a lead magnet generation strategy which worked well: hire a writer, include the right keywords, reach over a 5,000-word word count, and hit publish. Then, wait.

                  Nowadays, in-depth content requires more effort over time in order to pay off. Writing a big article, as good as it is, will not get your anywhere near the level of topical breadth required by Google to rank you first. Instead, your content marketing plan should be about having:

                  • a comprehensive pillar page covering a unique topic, and
                  • narrow-focused children articles to dig deeper.

                  Search engines also look at how often you publish about a specific topic, and when was the last time it was written about. Nobody likes a graveyard blog, it just makes the reader lose trust; as if the writer was not good enough, therefore had no traffic, before entirely giving up. Deep content requires a sustained effort on your part to always new find ways to write about a specific subject. Sure, it will be easy at first. But what about five years later? Well, you will still need to hit publish, all about the very same topics you already covered years ago.

                  Tools and platforms such as topicseed are here to help you find new article ideas pertaining to your core topics within a few clicks and a few minutes. The number of web pages, Wikipedia articles, and pieces of content, our machine-learning algorithms can analyze in seconds would take you months to digest. Our topicgraph finds closely related concepts in order for your domain to reach topical authority through content depth and content breadth.

                  ================================================ FILE: test/test-pages/tumblr/expected-metadata.json ================================================ { "title": "Minecraft 1.8 - The Bountiful Update", "byline": null, "dir": null, "lang": "en", "excerpt": "+ Added Granite, Andesite, and Diorite stone blocks, with smooth versions\n+ Added Slime Block\n+ Added Iron Trapdoor\n+ Added Prismarine and Sea Lantern blocks\n+ Added the Ocean Monument\n+ Added Red...", "siteName": "Minecraft Update News", "publishedTime": "2014-09-02T08:35:27-04:00", "readerable": true } ================================================ FILE: test/test-pages/tumblr/expected.html ================================================

                  Minecraft 1.8 - The Bountiful Update

                  + Added Granite, Andesite, and Diorite stone blocks, with smooth versions
                  + Added Slime Block
                  + Added Iron Trapdoor
                  + Added Prismarine and Sea Lantern blocks
                  + Added the Ocean Monument
                  + Added Red Sandstone
                  + Added Banners
                  + Added Armor Stands
                  + Added Coarse Dirt (dirt where grass won’t grow)
                  + Added Guardian mobs, with item drops
                  + Added Endermite mob
                  + Added Rabbits, with item drops
                  + Added Mutton and Cooked Mutton
                  + Villagers will harvest crops and plant new ones
                  + Mossy Cobblestone and Mossy Stone Bricks are now craftable
                  + Chiseled Stone Bricks are now craftable
                  + Doors and fences now come in all wood type variants
                  + Sponge block has regained its water-absorbing ability and becomes wet
                  + Added a spectator game mode (game mode 3)
                  + Added one new achievement
                  + Added “Customized” world type
                  + Added hidden “Debug Mode” world type
                  + Worlds can now have a world barrier
                  + Added @e target selector for Command Blocks
                  + Added /blockdata command
                  + Added /clone command
                  + Added /execute command
                  + Added /fill command
                  + Added /particle command
                  + Added /testforblocks command
                  + Added /title command
                  + Added /trigger command
                  + Added /worldborder command
                  + Added /stats command
                  + Containers can be locked in custom maps by using the “Lock” data tag
                  + Added logAdminCommands, showDeathMessages, reducedDebugInfo, sendCommandFeedback, and randomTickSpeed game rules
                  + Added three new statistics
                  + Player skins can now have double layers across the whole model, and left/right arms/legs can be edited independently
                  + Added a new player model with smaller arms, and a new player skin called Alex?
                  + Added options for configuring what pieces of the skin that are visible
                  + Blocks can now have custom visual variations in the resource packs
                  + Minecraft Realms now has an activity chart, so you can see who has been online
                  + Minecraft Realms now lets you upload your maps
                  * Difficulty setting is saved per world, and can be locked if wanted
                  * Enchanting has been redone, now costs lapis lazuli in addition to enchantment levels
                  * Villager trading has been rebalanced
                  * Anvil repairing has been rebalanced
                  * Considerable faster client-side performance
                  * Max render distance has been increased to 32 chunks (512 blocks)
                  * Adventure mode now prevents you from destroying blocks, unless your items have the CanDestroy data tag
                  * Resource packs can now also define the shape of blocks and items, and not just their textures
                  * Scoreboards have been given a lot of new features
                  * Tweaked the F3 debug screen
                  * Block ID numbers (such as 1 for stone), are being replaced by ID names (such as minecraft:stone)
                  * Server list has been improved
                  * A few minor changes to village and temple generation
                  * Mob heads for players now show both skin layers
                  * Buttons can now be placed on the ceiling
                  * Lots and lots of other changes
                  * LOTS AND LOTS of other changes
                  - Removed Herobrine

                  ================================================ FILE: test/test-pages/tumblr/source.html ================================================ Minecraft 1.8 - The Bountiful Update - Minecraft 1.8 - The Bountiful Update - Minecraft Update News

                  Minecraft News

                  		 Powered by Tumblr

                  Minecraft 1.8 - The Bountiful Update

                  + Added Granite, Andesite, and Diorite stone blocks, with smooth versions
                  + Added Slime Block
                  + Added Iron Trapdoor
                  + Added Prismarine and Sea Lantern blocks
                  + Added the Ocean Monument
                  + Added Red Sandstone
                  + Added Banners
                  + Added Armor Stands
                  + Added Coarse Dirt (dirt where grass won’t grow)
                  + Added Guardian mobs, with item drops
                  + Added Endermite mob
                  + Added Rabbits, with item drops
                  + Added Mutton and Cooked Mutton
                  + Villagers will harvest crops and plant new ones
                  + Mossy Cobblestone and Mossy Stone Bricks are now craftable
                  + Chiseled Stone Bricks are now craftable
                  + Doors and fences now come in all wood type variants
                  + Sponge block has regained its water-absorbing ability and becomes wet
                  + Added a spectator game mode (game mode 3)
                  + Added one new achievement
                  + Added “Customized” world type
                  + Added hidden “Debug Mode” world type
                  + Worlds can now have a world barrier
                  + Added @e target selector for Command Blocks
                  + Added /blockdata command
                  + Added /clone command
                  + Added /execute command
                  + Added /fill command
                  + Added /particle command
                  + Added /testforblocks command
                  + Added /title command
                  + Added /trigger command
                  + Added /worldborder command
                  + Added /stats command
                  + Containers can be locked in custom maps by using the “Lock” data tag
                  + Added logAdminCommands, showDeathMessages, reducedDebugInfo, sendCommandFeedback, and randomTickSpeed game rules
                  + Added three new statistics
                  + Player skins can now have double layers across the whole model, and left/right arms/legs can be edited independently
                  + Added a new player model with smaller arms, and a new player skin called Alex?
                  + Added options for configuring what pieces of the skin that are visible
                  + Blocks can now have custom visual variations in the resource packs
                  + Minecraft Realms now has an activity chart, so you can see who has been online
                  + Minecraft Realms now lets you upload your maps
                  * Difficulty setting is saved per world, and can be locked if wanted
                  * Enchanting has been redone, now costs lapis lazuli in addition to enchantment levels
                  * Villager trading has been rebalanced
                  * Anvil repairing has been rebalanced
                  * Considerable faster client-side performance
                  * Max render distance has been increased to 32 chunks (512 blocks)
                  * Adventure mode now prevents you from destroying blocks, unless your items have the CanDestroy data tag
                  * Resource packs can now also define the shape of blocks and items, and not just their textures
                  * Scoreboards have been given a lot of new features
                  * Tweaked the F3 debug screen
                  * Block ID numbers (such as 1 for stone), are being replaced by ID names (such as minecraft:stone)
                  * Server list has been improved
                  * A few minor changes to village and temple generation
                  * Mob heads for players now show both skin layers
                  * Buttons can now be placed on the ceiling
                  * Lots and lots of other changes
                  * LOTS AND LOTS of other changes
                  - Removed Herobrine

                  Official links:

                  Minecraft.net
                  Minecraft Realms
                  Minecraft on Facebook
                  Merchandise

                  Bug tracker
                  Account Support

                  Mojang on Twitter
                  Support on Twitter

                  Try our other games!

                  Community links:

                  Minecraft Forums
                  Minecraft on Twitch.tv
                  ================================================ FILE: test/test-pages/v8-blog/expected-metadata.json ================================================ { "title": "standalone WebAssembly binaries using Emscripten · V8", "byline": null, "dir": null, "lang": "en", "excerpt": "Emscripten now supports standalone Wasm files, which do not need JavaScript.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/v8-blog/expected.html ================================================

                  Emscripten has always focused first and foremost on compiling to the Web and other JavaScript environments like Node.js. But as WebAssembly starts to be used without JavaScript, new use cases are appearing, and so we've been working on support for emitting standalone Wasm files from Emscripten, that do not depend on the Emscripten JS runtime! This post explains why that's interesting.

                  Using standalone mode in Emscripten #

                  First, let's see what you can do with this new feature! Similar to this post let's start with a "hello world" type program that exports a single function that adds two numbers: 

                  // add.c
                  #include <emscripten.h>
                  EMSCRIPTEN_KEEPALIVE
                  int add(int x, int y) {
                    return x + y;
                  }

                  We'd normally build this with something like emcc -O3 add.c -o add.js which would emit add.js and add.wasm. Instead, let's ask emcc to only emit Wasm: 

                  emcc -O3 add.c -o add.wasm

                  When emcc sees we only want Wasm then it makes it "standalone" - a Wasm file that can run by itself as much as possible, without any JavaScript runtime code from Emscripten.

                  Disassembling it, it's very minimal - just 87 bytes! It contains the obvious add function 

                  (func $add (param $0 i32) (param $1 i32) (result i32)
                   (i32.add
                    (local.get $0)
                    (local.get $1)
                   )
                  )

                  and one more function, _start, 

                  (func $_start
                   (nop)
                  )

                  _start is part of the WASI spec, and Emscripten's standalone mode emits it so that we can run in WASI runtimes. (Normally _start would do global initialization, but here we just don't need any so it's empty.)

                  Write your own JavaScript loader #

                  One nice thing about a standalone Wasm file like this is that you can write custom JavaScript to load and run it, which can be very minimal depending on your use case. For example, we can do this in Node.js: 

                  // load-add.js
                  const binary = require('fs').readFileSync('add.wasm');
                  WebAssembly.instantiate(binary).then(({ instance }) => {
                    console.log(instance.exports.add(40, 2));
                  });

                  Just 4 lines! Running that prints 42 as expected. Note that while this example is very simplistic, there are cases where you simply don't need much JavaScript, and may be able to do better than Emscripten's default JavaScript runtime (which supports a bunch of environments and options). A real-world example of that is in zeux's meshoptimizer - just 57 lines, including memory management, growth, etc.!

                  Running in Wasm runtimes #

                  Another nice thing about standalone Wasm files is that you can run them in Wasm runtimes like wasmer, wasmtime, or WAVM. For example, consider this hello world: 

                  // hello.cpp
                  #include <stdio.h>
                  int main() {
                    printf("hello, world!\n");
                    return 0;
                  }

                  We can build and run that in any of those runtimes: 

                  $ emcc hello.cpp -O3 -o hello.wasm
                  $ wasmer run hello.wasm
                  hello, world!
                  $ wasmtime hello.wasm
                  hello, world!
                  $ wavm run hello.wasm
                  hello, world!

                  Emscripten uses WASI APIs as much as possible, so programs like this end up using 100% WASI and can run in WASI-supporting runtimes (see notes later on what programs require more than WASI).

                  Building Wasm plugins #

                  Aside from the Web and the server, an exciting area for Wasm is plugins. For example, an image editor might have Wasm plugins that can perform filters and other operations on the image. For that type of use case you want a standalone Wasm binary, just like in the examples so far, but where it also has a proper API for the embedding application.

                  Plugins are sometimes related to dynamic libraries, as dynamic libraries are one way to implement them. Emscripten has support for dynamic libraries with the SIDE_MODULE option, and this has been a way to build Wasm plugins. The new standalone Wasm option described here is an improvement on that in several ways: First, a dynamic library has relocatable memory, which adds overhead if you don’t need it (and you don’t if you aren’t linking the Wasm with another Wasm after loading it). Second, standalone output is designed to run in Wasm runtimes as well, as mentioned earlier.

                  Okay, so far so good: Emscripten can either emit JavaScript + WebAssembly as it always did, and now it can also emit just WebAssembly by itself, which lets you run it in places that don't have JavaScript like Wasm runtimes, or you can write your own custom JavaScript loader code, etc. Now let's talk about the background and the technical details!

                  WebAssembly's two standard APIs #

                  WebAssembly can only access the APIs it receives as imports - the core Wasm spec has no concrete API details. Given the current trajectory of Wasm, it looks like there will be 3 main categories of APIs that people import and use:

                  • Web APIs: This is what Wasm programs use on the Web, which are the existing standardized APIs that JavaScript can use too. Currently these are called indirectly, through JS glue code, but in the future with interface types they will be called directly.
                  • WASI APIs: WASI focuses on standardizing APIs for Wasm on the server.
                  • Other APIs: Various custom embeddings will define their own application-specific APIs. For example, we gave the example earlier of an image editor with Wasm plugins that implement an API to do visual effects. Note that a plugin might also have access to “system” APIs, like a native dynamic library would, or it might be very sandboxed and have no imports at all (if the embedding just calls its methods).

                  WebAssembly is in the interesting position of having two standardized sets of APIs. This does makes sense in that one is for the Web and one for the server, and those environments do have different requirements; for similar reasons Node.js does not have identical APIs to JavaScript on the Web.

                  However, there is more than the Web and the server, in particular there are also Wasm plugins. For one thing, plugins can run inside an application that may be on the Web (just like JS plugins) or off the Web; for another, regardless of where the embedding application is, a plugin environment is not a Web nor a server environment. So it's not immediately obvious which sets of APIs will be used - it may depend on the code being ported, the Wasm runtime being embedded, etc.

                  Let's unify as much as possible #

                  One concrete way Emscripten hopes to help here is that by using WASI APIs as much as possible we can avoid unnecessary API differences. As mentioned earlier, on the Web Emscripten code accesses Web APIs indirectly, through JavaScript, so where that JavaScript API could look like WASI, we'd be removing an unnecessary API difference, and that same binary can also run on the server. In other words, if Wasm wants to log some info, it needs to call into JS, something like this: 

                  wasm   =>   function musl_writev(..) { .. console.log(..) .. }

                  musl_writev is an implementation of the Linux syscall interface that musl libc uses to write data to a file descriptor, and that ends up calling console.log with the proper data. The Wasm module imports and calls that musl_writev, which defines an ABI between the JS and the Wasm. That ABI is arbitrary (and in fact Emscripten has changed its ABI over time to optimize it). If we replace that with an ABI that matches WASI, we can get this: 

                  wasm   =>   function __wasi_fd_write(..) { .. console.log(..) .. }

                  This isn't a big change, just requiring some refactoring of the ABI, and when running in a JS environment it doesn't matter much. But now the Wasm can run without the JS since that WASI API is recognized by WASI runtimes! That’s how the standalone Wasm examples from before work, just by refactoring Emscripten to use WASI APIs.

                  Another advantage of Emscripten using WASI APIs is that we can help the WASI spec by finding real-world issues. For example, we found that changing the WASI "whence" constants would be useful, and we've started some discussions around code size and POSIX compatibility.

                  Emscripten using WASI as much as possible is also useful in that it lets users use a single SDK to target Web, server, and plugin environments. Emscripten isn't the only SDK allowing that, as the WASI SDK's output can be run on the Web using the WASI Web Polyfill or Wasmer's wasmer-js, but Emscripten’s Web output is more compact, so it lets a single SDK be used without compromising Web performance.

                  Speaking of which, you can emit a standalone Wasm file from Emscripten with optional JS in a single command: 

                  emcc -O3 add.c -o add.js -s STANDALONE_WASM

                  That emits add.js and add.wasm. The Wasm file is standalone just like earlier when we only emitted a Wasm file by itself (STANDALONE_WASM was set automatically when we said -o add.wasm), but now in addition there is a JS file that can load and run it. The JS is useful for running it on the Web if you don't want to write your own JS for that.

                  Do we need non-standalone Wasm? #

                  Why does the STANDALONE_WASM flag exist? In theory Emscripten could always set STANDALONE_WASM, which would be simpler. But standalone Wasm files can't depend on JS, and that has some downsides:

                  • We can't minify the Wasm import and export names, as the minification only works if both sides agree, the Wasm and what loads it.
                  • Normally we create the Wasm Memory in JS so that JS can start to use it during startup, which lets us do work in parallel. But in standalone Wasm we have to create the Memory in the Wasm.
                  • Some APIs are just easy to do in JS. For example __assert_fail, which is called when a C assertion fails, is normally implemented in JS. It takes just a single line, and even if you include the JS functions it calls, the total code size is quite small. On the other hand, in a standalone build we can't depend on JS, so we use musl's assert.c. That uses fprintf, which means it ends up pulling in a bunch of C stdio support, including things with indirect calls that make it hard to remove unused functions. Overall, there are many such details that end up making a difference in total code size.

                  If you want to run both on the Web and elsewhere, and you want 100% optimal code size and startup times, you should make two separate builds, one with -s STANDALONE and one without. That's very easy as it's just flipping one flag!

                  Necessary API differences #

                  We saw that Emscripten uses WASI APIs as much as possible to avoid unnecessary API differences. Are there any necessary ones? Sadly, yes - some WASI APIs require tradeoffs. For example:

                  • WASI does not support various POSIX features, like user/group/world file permissions, as a result of which you can't fully implement a (Linux) system ls for example (see details in that link). Emscripten's existing filesystem layer does support some of those things, so if we switched to WASI APIs for all filesystem operations then we'd be losing some POSIX support.
                  • WASI's path_open has a cost in code size because it forces extra permissions handling in the Wasm itself. That code is unnecessary on the Web.
                  • WASI doesn't provide a notification API for memory growth, and as a result, JS runtimes must constantly check if memory grew and if so update their views, on every import and export. To avoid that overhead, Emscripten provides a notification API, emscripten_notify_memory_growth, which you can see implemented in a single line in zeux's meshoptimizer that we mentioned earlier.

                  In time WASI may add more POSIX support, a memory growth notification, etc. - WASI is still highly experimental and expected to change significantly. For now, to avoid regressions in Emscripten we do not emit 100% WASI binaries if you use certain features. In particular, opening files uses a POSIX method instead of WASI, which means that if you call fopen then the resulting Wasm file will not be 100% WASI - however, if all you do is use printf, which operates on the already-open stdout, then it will be 100% WASI, as in the "hello world" example we saw near the beginning, where Emscripten's output does run in WASI runtimes.

                  If it would be useful for users we can add a PURE_WASI option which would sacrifice code size in return for strict WASI compliance, but if that's not urgent (and most plugin use cases we’ve seen so far don’t need full file I/O) then maybe we can wait for WASI to improve to where Emscripten can remove these non-WASI APIs. That would be the best outcome, and we’re working towards that as you can see in the links above.

                  However, even if WASI does improve, there is no avoiding the fact that Wasm has two standardized APIs as mentioned earlier. In the future I expect Emscripten will call Web APIs directly using interface types, because that will be more compact than calling a WASI-looking JS API that then calls a Web API (as in the musl_writev example from before). We could have a polyfill or a translation layer of some sort to help here, but we wouldn't want to use it unnecessarily, so we will need separate builds for Web and WASI environments. (This is somewhat unfortunate; in theory this could have been avoided if WASI were a superset of Web APIs, but obviously that would have meant compromises on the server side.)

                  Current status #

                  Quite a lot works already! The main limitations are:

                  • WebAssembly limitations: Various features, like C++ exceptions, setjmp, and pthreads, depend on JavaScript due to Wasm limitations, and there is no good non-JS replacement yet. (Emscripten may start to support some of them using Asyncify, or maybe we'll just wait for native Wasm features to arrive to VMs.)
                  • WASI limitations: Libraries and APIs like OpenGL and SDL don't have corresponding WASI APIs yet.

                  You can still use all those in Emscripten's standalone mode, but the output will contain calls to JS runtime support code. As a result, it will not be 100% WASI (for similar reasons those features also do not work in the WASI SDK). Those Wasm files won't run in WASI runtimes, but you can use them on the Web and you can write your own JS runtime for them. You can also use them as plugins; for example, a game engine could have plugins that render using OpenGL, and the developer would compile them in standalone mode and then implement the OpenGL imports in the engine's Wasm runtime. Standalone Wasm mode still helps here because it makes the output as standalone as Emscripten can make it.

                  You may also find APIs that do have a non-JS replacement that we haven’t converted yet, as work is still ongoing. Please file bugs, and as always help is welcome!

                  ================================================ FILE: test/test-pages/v8-blog/source.html ================================================ Outside the web: standalone WebAssembly binaries using Emscripten · V8

                  Outside the web: standalone WebAssembly binaries using Emscripten

                  Published · Tagged with WebAssembly tooling

                  Emscripten has always focused first and foremost on compiling to the Web and other JavaScript environments like Node.js. But as WebAssembly starts to be used without JavaScript, new use cases are appearing, and so we've been working on support for emitting standalone Wasm files from Emscripten, that do not depend on the Emscripten JS runtime! This post explains why that's interesting.

                  Using standalone mode in Emscripten #

                  First, let's see what you can do with this new feature! Similar to this post let's start with a "hello world" type program that exports a single function that adds two numbers: 

                  // add.c
                  #include <emscripten.h>

                  EMSCRIPTEN_KEEPALIVE
                  int add(int x, int y) {
                    return x + y;
                  }

                  We'd normally build this with something like emcc -O3 add.c -o add.js which would emit add.js and add.wasm. Instead, let's ask emcc to only emit Wasm: 

                  emcc -O3 add.c -o add.wasm

                  When emcc sees we only want Wasm then it makes it "standalone" - a Wasm file that can run by itself as much as possible, without any JavaScript runtime code from Emscripten.

                  Disassembling it, it's very minimal - just 87 bytes! It contains the obvious add function 

                  (func $add (param $0 i32) (param $1 i32) (result i32)
                   (i32.add
                    (local.get $0)
                    (local.get $1)
                   )
                  )

                  and one more function, _start, 

                  (func $_start
                   (nop)
                  )

                  _start is part of the WASI spec, and Emscripten's standalone mode emits it so that we can run in WASI runtimes. (Normally _start would do global initialization, but here we just don't need any so it's empty.)

                  Write your own JavaScript loader #

                  One nice thing about a standalone Wasm file like this is that you can write custom JavaScript to load and run it, which can be very minimal depending on your use case. For example, we can do this in Node.js: 

                  // load-add.js
                  const binary = require('fs').readFileSync('add.wasm');

                  WebAssembly.instantiate(binary).then(({ instance }) => {
                    console.log(instance.exports.add(40, 2));
                  });

                  Just 4 lines! Running that prints 42 as expected. Note that while this example is very simplistic, there are cases where you simply don't need much JavaScript, and may be able to do better than Emscripten's default JavaScript runtime (which supports a bunch of environments and options). A real-world example of that is in zeux's meshoptimizer - just 57 lines, including memory management, growth, etc.!

                  Running in Wasm runtimes #

                  Another nice thing about standalone Wasm files is that you can run them in Wasm runtimes like wasmer, wasmtime, or WAVM. For example, consider this hello world: 

                  // hello.cpp
                  #include <stdio.h>

                  int main() {
                    printf("hello, world!\n");
                    return 0;
                  }

                  We can build and run that in any of those runtimes: 

                  $ emcc hello.cpp -O3 -o hello.wasm
                  $ wasmer run hello.wasm
                  hello, world!
                  $ wasmtime hello.wasm
                  hello, world!
                  $ wavm run hello.wasm
                  hello, world!

                  Emscripten uses WASI APIs as much as possible, so programs like this end up using 100% WASI and can run in WASI-supporting runtimes (see notes later on what programs require more than WASI).

                  Building Wasm plugins #

                  Aside from the Web and the server, an exciting area for Wasm is plugins. For example, an image editor might have Wasm plugins that can perform filters and other operations on the image. For that type of use case you want a standalone Wasm binary, just like in the examples so far, but where it also has a proper API for the embedding application.

                  Plugins are sometimes related to dynamic libraries, as dynamic libraries are one way to implement them. Emscripten has support for dynamic libraries with the SIDE_MODULE option, and this has been a way to build Wasm plugins. The new standalone Wasm option described here is an improvement on that in several ways: First, a dynamic library has relocatable memory, which adds overhead if you don’t need it (and you don’t if you aren’t linking the Wasm with another Wasm after loading it). Second, standalone output is designed to run in Wasm runtimes as well, as mentioned earlier.

                  Okay, so far so good: Emscripten can either emit JavaScript + WebAssembly as it always did, and now it can also emit just WebAssembly by itself, which lets you run it in places that don't have JavaScript like Wasm runtimes, or you can write your own custom JavaScript loader code, etc. Now let's talk about the background and the technical details!

                  WebAssembly's two standard APIs #

                  WebAssembly can only access the APIs it receives as imports - the core Wasm spec has no concrete API details. Given the current trajectory of Wasm, it looks like there will be 3 main categories of APIs that people import and use:

                  • Web APIs: This is what Wasm programs use on the Web, which are the existing standardized APIs that JavaScript can use too. Currently these are called indirectly, through JS glue code, but in the future with interface types they will be called directly.
                  • WASI APIs: WASI focuses on standardizing APIs for Wasm on the server.
                  • Other APIs: Various custom embeddings will define their own application-specific APIs. For example, we gave the example earlier of an image editor with Wasm plugins that implement an API to do visual effects. Note that a plugin might also have access to “system” APIs, like a native dynamic library would, or it might be very sandboxed and have no imports at all (if the embedding just calls its methods).

                  WebAssembly is in the interesting position of having two standardized sets of APIs. This does makes sense in that one is for the Web and one for the server, and those environments do have different requirements; for similar reasons Node.js does not have identical APIs to JavaScript on the Web.

                  However, there is more than the Web and the server, in particular there are also Wasm plugins. For one thing, plugins can run inside an application that may be on the Web (just like JS plugins) or off the Web; for another, regardless of where the embedding application is, a plugin environment is not a Web nor a server environment. So it's not immediately obvious which sets of APIs will be used - it may depend on the code being ported, the Wasm runtime being embedded, etc.

                  Let's unify as much as possible #

                  One concrete way Emscripten hopes to help here is that by using WASI APIs as much as possible we can avoid unnecessary API differences. As mentioned earlier, on the Web Emscripten code accesses Web APIs indirectly, through JavaScript, so where that JavaScript API could look like WASI, we'd be removing an unnecessary API difference, and that same binary can also run on the server. In other words, if Wasm wants to log some info, it needs to call into JS, something like this: 

                  wasm   =>   function musl_writev(..) { .. console.log(..) .. }

                  musl_writev is an implementation of the Linux syscall interface that musl libc uses to write data to a file descriptor, and that ends up calling console.log with the proper data. The Wasm module imports and calls that musl_writev, which defines an ABI between the JS and the Wasm. That ABI is arbitrary (and in fact Emscripten has changed its ABI over time to optimize it). If we replace that with an ABI that matches WASI, we can get this: 

                  wasm   =>   function __wasi_fd_write(..) { .. console.log(..) .. }

                  This isn't a big change, just requiring some refactoring of the ABI, and when running in a JS environment it doesn't matter much. But now the Wasm can run without the JS since that WASI API is recognized by WASI runtimes! That’s how the standalone Wasm examples from before work, just by refactoring Emscripten to use WASI APIs.

                  Another advantage of Emscripten using WASI APIs is that we can help the WASI spec by finding real-world issues. For example, we found that changing the WASI "whence" constants would be useful, and we've started some discussions around code size and POSIX compatibility.

                  Emscripten using WASI as much as possible is also useful in that it lets users use a single SDK to target Web, server, and plugin environments. Emscripten isn't the only SDK allowing that, as the WASI SDK's output can be run on the Web using the WASI Web Polyfill or Wasmer's wasmer-js, but Emscripten’s Web output is more compact, so it lets a single SDK be used without compromising Web performance.

                  Speaking of which, you can emit a standalone Wasm file from Emscripten with optional JS in a single command: 

                  emcc -O3 add.c -o add.js -s STANDALONE_WASM

                  That emits add.js and add.wasm. The Wasm file is standalone just like earlier when we only emitted a Wasm file by itself (STANDALONE_WASM was set automatically when we said -o add.wasm), but now in addition there is a JS file that can load and run it. The JS is useful for running it on the Web if you don't want to write your own JS for that.

                  Do we need non-standalone Wasm? #

                  Why does the STANDALONE_WASM flag exist? In theory Emscripten could always set STANDALONE_WASM, which would be simpler. But standalone Wasm files can't depend on JS, and that has some downsides:

                  • We can't minify the Wasm import and export names, as the minification only works if both sides agree, the Wasm and what loads it.
                  • Normally we create the Wasm Memory in JS so that JS can start to use it during startup, which lets us do work in parallel. But in standalone Wasm we have to create the Memory in the Wasm.
                  • Some APIs are just easy to do in JS. For example __assert_fail, which is called when a C assertion fails, is normally implemented in JS. It takes just a single line, and even if you include the JS functions it calls, the total code size is quite small. On the other hand, in a standalone build we can't depend on JS, so we use musl's assert.c. That uses fprintf, which means it ends up pulling in a bunch of C stdio support, including things with indirect calls that make it hard to remove unused functions. Overall, there are many such details that end up making a difference in total code size.

                  If you want to run both on the Web and elsewhere, and you want 100% optimal code size and startup times, you should make two separate builds, one with -s STANDALONE and one without. That's very easy as it's just flipping one flag!

                  Necessary API differences #

                  We saw that Emscripten uses WASI APIs as much as possible to avoid unnecessary API differences. Are there any necessary ones? Sadly, yes - some WASI APIs require tradeoffs. For example:

                  • WASI does not support various POSIX features, like user/group/world file permissions, as a result of which you can't fully implement a (Linux) system ls for example (see details in that link). Emscripten's existing filesystem layer does support some of those things, so if we switched to WASI APIs for all filesystem operations then we'd be losing some POSIX support.
                  • WASI's path_open has a cost in code size because it forces extra permissions handling in the Wasm itself. That code is unnecessary on the Web.
                  • WASI doesn't provide a notification API for memory growth, and as a result, JS runtimes must constantly check if memory grew and if so update their views, on every import and export. To avoid that overhead, Emscripten provides a notification API, emscripten_notify_memory_growth, which you can see implemented in a single line in zeux's meshoptimizer that we mentioned earlier.

                  In time WASI may add more POSIX support, a memory growth notification, etc. - WASI is still highly experimental and expected to change significantly. For now, to avoid regressions in Emscripten we do not emit 100% WASI binaries if you use certain features. In particular, opening files uses a POSIX method instead of WASI, which means that if you call fopen then the resulting Wasm file will not be 100% WASI - however, if all you do is use printf, which operates on the already-open stdout, then it will be 100% WASI, as in the "hello world" example we saw near the beginning, where Emscripten's output does run in WASI runtimes.

                  If it would be useful for users we can add a PURE_WASI option which would sacrifice code size in return for strict WASI compliance, but if that's not urgent (and most plugin use cases we’ve seen so far don’t need full file I/O) then maybe we can wait for WASI to improve to where Emscripten can remove these non-WASI APIs. That would be the best outcome, and we’re working towards that as you can see in the links above.

                  However, even if WASI does improve, there is no avoiding the fact that Wasm has two standardized APIs as mentioned earlier. In the future I expect Emscripten will call Web APIs directly using interface types, because that will be more compact than calling a WASI-looking JS API that then calls a Web API (as in the musl_writev example from before). We could have a polyfill or a translation layer of some sort to help here, but we wouldn't want to use it unnecessarily, so we will need separate builds for Web and WASI environments. (This is somewhat unfortunate; in theory this could have been avoided if WASI were a superset of Web APIs, but obviously that would have meant compromises on the server side.)

                  Current status #

                  Quite a lot works already! The main limitations are:

                  • WebAssembly limitations: Various features, like C++ exceptions, setjmp, and pthreads, depend on JavaScript due to Wasm limitations, and there is no good non-JS replacement yet. (Emscripten may start to support some of them using Asyncify, or maybe we'll just wait for native Wasm features to arrive to VMs.)
                  • WASI limitations: Libraries and APIs like OpenGL and SDL don't have corresponding WASI APIs yet.

                  You can still use all those in Emscripten's standalone mode, but the output will contain calls to JS runtime support code. As a result, it will not be 100% WASI (for similar reasons those features also do not work in the WASI SDK). Those Wasm files won't run in WASI runtimes, but you can use them on the Web and you can write your own JS runtime for them. You can also use them as plugins; for example, a game engine could have plugins that render using OpenGL, and the developer would compile them in standalone mode and then implement the OpenGL imports in the engine's Wasm runtime. Standalone Wasm mode still helps here because it makes the output as standalone as Emscripten can make it.

                  You may also find APIs that do have a non-JS replacement that we haven’t converted yet, as work is still ongoing. Please file bugs, and as always help is welcome!

                  ================================================ FILE: test/test-pages/videos-1/expected-metadata.json ================================================ { "title": "How to watch the 21 best films of 2017", "byline": "Alissa Wilkinson", "dir": null, "excerpt": "It was an extraordinary year for movies.", "siteName": "Vox", "publishedTime": "2017-12-15T08:50:02-05:00", "readerable": true } ================================================ FILE: test/test-pages/videos-1/expected.html ================================================

                  In the introduction to her review anthology For Keeps: 30 Years at the Movies, the legendary film critic Pauline Kael wrote, “I’m frequently asked why I don’t write my memoirs. I think I have.” She meant what most movie critics realize at some point: that reading your past reviews and revisiting the lists of films you liked most during the year reveals not just something about a particular year in cinema, but something about you as well.

                  That’s the feeling I get constructing my list of the best films of 2017, a year that overflowed with great films in every genre, from horror and romantic comedy to documentary and arthouse drama. Some of the films on my list have commonalities — ghosts, meditations on memory and interpersonal connection, and women who refuse to behave — but mostly they underscore just how vibrant cinema remains as an art form, even in the midst of massive cultural shifts in the industry and beyond. And it is a keen reminder to me of all the 2017 conversations I’ve had around and at the movies — and the ways I will never be the same.

                  Here are my top 21 films of 2017 and how to watch them at home, with 14 honorable mentions.

                  21) Star Wars: The Last Jedi

                  I am as shocked as anyone that a Star Wars movie found its way onto my list — but I was bowled over by The Last Jedi, which may be one of the series’ best. In the hands of writer-director Rian Johnson (who will also oversee a new Star Wars trilogy), The Last Jedi is beautiful to look at and keeps its eye on the relationships between characters and how they communicate with one another, in addition to the bigger galactic story. The same characters are back, but they seem infused with new life, and the galaxy with a new kind of hope. The movie’s best details are in the strong bonds that develop between characters, and I left the film with the realization that for the first time in my life, I loved a Star Wars movie. Now I understand the magic.

                  Star Wars: The Last Jedi is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  20) Faces Places

                  The unusual documentary Faces Places (in French, Visages Villages) turns on the friendship between the accomplished street artist JR and legendary film director Agnès Varda, whose work was central to the development of the French New Wave movement. The pair (whose difference in age is 55 years) met after years of admiring each other’s work and decided to create a documentary portrait of France — by making a number of actual portraits. The film chronicles a leg of the "Inside Outside Project," a roving art initiative in which JR makes enormous portraits of people he meets and pastes them onto buildings and walls. In the film, Varda joins him, and as they talk to people around the country, they grow in their understanding of themselves and of each other. The development of their friendship, which is both affectionate and mutually sharpening, forms Faces Places’ emotional center.

                  Faces Places is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  19) Ingrid Goes West

                  Ingrid Goes West is a twisted and dark comedy — part addiction narrative, part stalker story — and yet it’s set in a world that’s almost pathologically cheery: the glossy, sunny, nourishing, superfood- and superlative-loving universe of Instagram celebrity. But despite Ingrid Goes West’s spot-on take on that world, the best thing about the film is that it refuses to traffic in lazy buzzwords and easy skewering, particularly at the expense of young women. Instead, the movie conveys that behind every Instagram image and meltdown is a real person, with real insecurities, real feelings, and real problems. And it recognizes that living a life performed in public can be its own kind of self-deluding prison.

                  Ingrid Goes West is currently streaming on Hulu and available to digitally rent on YouTube and Google Play.

                  18) Lady Macbeth

                  Lady Macbeth is no placid costume drama. Adapted from an 1865 Russian novella by Nikolai Leskov, the movie follows Katherine (the astounding Florence Pugh), a woman in the Lady Macbeth line characterized by a potent cocktail of very few scruples and a lot of determination. She's a chilling avatar for the ways that class and privilege — both obvious and hidden — insulate some people from the consequences of their actions while damning others. Lady Macbeth is also a dazzling directorial debut from William Oldroyd, a thrilling combination of sex, murder, intrigue, and power plays. It’s visually stunning, each frame composed so carefully and deliberately that the wildness and danger roiling just below the surface feels even more frightening. Each scene ratchets up the tension to an explosive, chilling end.

                  Lady Macbeth is currently streaming on HBO Go and HBO Now, and it is available to digitally rent on Amazon Prime, Vudu, YouTube, iTunes, and Google Play.

                  17) BPM (Beats Per Minute)

                  BPM (Beats Per Minute) is a remarkably tender and stirring story of the Paris chapter of ACT UP, an AIDS activism group, and the young people who found themselves caught in the crosshairs of the AIDS crisis in the early 1990s. The film follows both the group's actions and the individual members’ shifting relationships to one another — enemies becoming friends, friends becoming lovers, lovers becoming caretakers — as well as their struggles with the disease wracking their community. As an account of the period, it’s riveting; as an exploration of life and love set at the urgent intersection of the political and the personal, it’s devastating.

                  BPM (Beats Per Minute) is currently streaming on Hulu and available to digitally rent on Google Play and YouTube.

                  16) The Big Sick

                  Few 2017 movies could top the charm and tenderness of The Big Sick, which hits all the right romantic comedy notes with one unusual distinction: It feels like real life. That’s probably because The Big Sick is written by real-life married couple Emily V. Gordon and Silicon Valley's Kumail Nanjiani, and based on their real-life romance. The Big Sick — which stars Nanjiani as a version of himself, alongside Zoe Kazan as Emily — is funny and sweet while not backing away from matters that romantic comedies don’t usually touch on, like serious illness, struggles in long-term marriages, and religion. As it tells the couple’s story, which takes a serious turn when Emily falls ill with a mysterious infection and her parents (played by Holly Hunter and Ray Romano) come to town, it becomes a funny and wise story about real love.

                  The Big Sick is currently streaming on Amazon Prime and available to digitally rent on iTunes, Vudu, Amazon, YouTube, and Google Play.

                  15) Mother!

                  There’s so much pulsing beneath the surface of Mother! that it’s hard to grab on to just one theme as what it “means.” It’s full-on apocalyptic fiction, and like all stories of apocalypse, it’s intended to draw back the veil on reality and show us what’s really beneath. And this movie gets wild: If its gleeful cracking apart of traditional theologies doesn’t get you (there’s a lot of Catholic folk imagery here, complete with an Ash Wednesday-like mud smearing on the foreheads of the faithful), its bonkers scenes of chaos probably will. Mother! is a movie designed to provoke fury, ecstasy, madness, catharsis, and more than a little awe. Watching it, and then participating in the flurry of arguments and discussions unpacking it, was among my best moviegoing experiences of 2017.

                  Mother! is available to digitally purchase on Google Play and YouTube.

                  14) A Ghost Story

                  Director David Lowery filmed A Ghost Story in secret, then premiered it at the Sundance Film Festival to critical acclaim. The movie starts out being about a grieving widow (Rooney Mara) trying to live through the pain of losing her beloved husband, but it soon shifts focus to the ghost of her husband (Casey Affleck, covered in a sheet), evolving into a compelling rumination on the nature of time, memory, history, and the universe. Bathed in warm humor and wistful longing, it's a film that stays with you long after it’s over, a lingering reminder of the inextricable link between love and place.

                  A Ghost Story is available to digitally rent on iTunes, Vudu, Amazon, Google Play, and YouTube.

                  13) The Square

                  The Square is currently streaming on Hulu and available to digitally rent on Google Play and YouTube.

                  12) Dunkirk

                  Dunkirk, a true cinematic achievement from acclaimed director Christopher Nolan, backs off conventional notions of narrative and chronology as much as possible, while leaning headfirst into everything else that makes a movie a visceral work of art aimed at the senses: the images, the sounds, the scale, the swelling vibrations of it all. You can’t smell the sea spray, but your brain may trick you into thinking you can. Nolan’s camera pushes the edges of the screen as far as it can as Dunkirk engulfs the audience in something that feels like a lot more than a war movie. It’s a symphony for the brave and broken, and it resolves in a major key — but one with an undercurrent of sorrow, and of sober warning. Courage in the face of danger is not just for characters in movies.

                  Dunkirk is currently streaming on HBO Go and HBO Now, and available to digitally rent on Google Play and YouTube.

                  11) Rat Film

                  Rat Film is about rats, yes — and rat poison experts and rat hunters and people who keep rats as pets. But it’s also about the history of eugenics, dubious science, “redlining,” and segregated housing in Baltimore. All these pieces come together to form one big essay, where the meaning of each vignette only becomes clearer in light of the whole. It’s a fast-paced, no-holds-barred exploration of a damning history, and it accrues meaning as the images, sounds, and text pile up.

                  Rat Film is available to digitally rent on YouTube and Google Play.

                  10) A Quiet Passion

                  A Quiet Passion is technically a biographical film about Emily Dickinson, but it transcends its genre to become something more like poetry. It’s a perplexing and challenging film, crafted without the traditional guardrails that guide most biographical movies — dates, times, major accomplishments, and so on. Time slips away in the film almost imperceptibly, and the narrative arc doesn’t yield easily to the viewer. Cynthia Nixon plays Emily Dickinson, whose poetry and life is a perfect match for the signature style of director Terence Davies: rich in detail, deeply enigmatic, and weighed down with a kind of sparkling, joy-tinged sorrow. A Quiet Passion is a portrait, both visual and narrative, of the kind of saint most modern people can understand: one who is certain of her uncertainty, and yearning to walk the path on which her passion and longing meet.

                  A Quiet Passion is currently streaming on Amazon Prime and available to digitally rent or purchase on iTunes, Vudu, Amazon, YouTube, and Google Play.

                  9) Columbus

                  Columbus is a stunner of a debut from video essayist turned director Kogonada. Haley Lu Richardson stars as Casey, a young woman living in Columbus, Indiana, who cares for her mother, works at a library, and harbors a passion for architecture. (Columbus is a mecca for modernist architecture scholars and enthusiasts.) When a visiting architecture scholar falls into a coma in Columbus, his estranged son Jin (John Cho) arrives to wait for him and strikes up a friendship with Casey, who starts to show him her favorite buildings. The two begin to unlock something in each other that’s hard to define but life-changing for both. Columbus is beautiful and subtle, letting us feel how the places we build and the people we let near us move and mold us.

                  Columbus is currently streaming on Hulu and available to rent on Google Play and YouTube.

                  8) The Florida Project

                  Sean Baker’s The Florida Project unfolds at first like a series of sketches about the characters who live in a purple-painted, $35-a-night motel called the Magic Castle down the street from Disney World. The film is held together by the hysterical antics of a kid named Moonee and her pack of young friends, as well as long-suffering hotel manager Bobby (a splendid, warm Willem Dafoe), who tries to put up with it all while keeping some kind of order. But as The Florida Project goes on, a narrative starts to form, one that chronicles with heartbreaking attention the sort of dilemmas that face poor parents and their children in America, and the broken systems that try to cope with impossible situations.

                  The Florida Project is currently streaming on Amazon Prime and available to digitally rent on YouTube, Vudu, and Google Play.

                  7) Call Me by Your Name

                  Luca Guadagnino’s gorgeous film Call Me by Your Name adapts André Aciman’s 2007 novel about a precocious 17-year-old named Elio (Timothée Chalamet), who falls in lust and love with his father’s 24-year-old graduate student Oliver (Armie Hammer). It’s remarkable for how it turns literature into pure cinema, all emotion and image and heady sensation. Set in 1983 in Northern Italy, Call Me by Your Name is less about coming out than coming of age, but it also captures a particular sort of love that’s equal parts passion and torment, a kind of irrational heart fire that opens a gate into something longer-lasting. The film is a lush, heady experience for the body, but it’s also an arousal for the soul.

                  Call Me By Your Name is available to digitally purchase on Amazon, YouTube, and Google Play.

                  6) Personal Shopper

                  In her second collaboration with French director Olivier Assayas, Kristen Stewart plays a personal shopper to a wealthy socialite, with a sideline as an amateur ghost hunter who’s searching for her dead twin brother. Personal Shopper is deeper than it seems at first blush, a meditation on grief and an exploration of “between” places — on the fringes of wealth, and in the space between life and death. Some souls are linked in a way that can’t be shaken, and whether or not there’s an afterlife doesn’t change the fact that we see and sense them everywhere. (Personal Shopper also has one of the most tense extended scenes involving text messaging ever seen onscreen.)

                  Personal Shopper is currently streaming on Showtime and available to rent on Vudu, YouTube, Amazon, iTunes, and Google Play.

                  5) Princess Cyd

                  Stephen Cone is a master of small, carefully realized filmmaking; his earlier films such as The Wise Kids and Henry Gamble’s Birthday Party combine an unusual level of empathy for his characters with an unusual combination of interests: love, desire, sexual awakenings, and religion. Princess Cyd is his most accomplished film yet, about a young woman named Cyd (Jessie Pinnick) who finds herself attracted to Katie (Malic White), a barista, while visiting her Aunt Miranda (Rebecca Spence, playing a character modeled on the author Marilynne Robinson) in Chicago. As she works through her own sexual awakening with Katie, Cyd unwinds some of the ways Miranda’s life has gotten too safe. They provoke each other while forming a bond and being prodded toward a bigger understanding of the world. It is a graceful and honest film, and it feels like a modest miracle.

                  Princess Cyd is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  4) Get Out

                  Racism is sinister, frightening, and deadly. But Get Out (a stunning directorial debut from Key & Peele's Jordan Peele) isn’t about the blatantly, obviously scary kind of racism — burning crosses and lynchings and snarling hate. Instead, it’s interested in showing how the parts of racism that try to be aggressively unscary are just as horrifying, and it’s interested in making us feel that horror in a visceral, bodily way. In the tradition of the best classic social thrillers, Get Out takes a topic that is often approached cerebrally — casual racism — and turns it into something you feel in your tummy. And it does it with a wicked sense of humor.

                  Get Out is currently streaming on HBO Go and HBO Now, and is available to digitally rent on iTunes, Amazon, Google Play, YouTube, and Vudu.

                  3) The Work

                  The Work is an outstanding, astonishing accomplishment and a viewing experience that will leave you shaken (but in a good way). At Folsom Prison in California, incarcerated men regularly participate in group therapy, and each year other men from the “outside” apply to participate in an intense four-day period of group therapy alongside Folsom’s inmates. The Work spends almost all of its time inside the room where that therapy happens, observing the strong, visceral, and sometimes violent emotions the men feel as they expose the hurt and raw nerves that have shaped how they encounter the world. Watching is not always easy, but by letting us peek in, the film invites viewers to become part of the experience — as if we, too, are being asked to let go.

                  The Work is streaming on Topic.com and available to digitally rent on Google Play and YouTube.

                  2) Ex Libris

                  Frederick Wiseman is one of the towering giants of nonfiction film, a keen observer of American institutions — ranging from prisons to dance companies to welfare offices — for the past half-century. Ex Libris is his mesmerizing look at the New York Public Library and the many functions it fills, which go far beyond housing books. Wiseman works in the observational mode, which means his films contain no captions, dates, or talking-head interviews: We just see what his camera captured, which in this case includes community meetings, benefit dinners, after-school programs, readings with authors and scholars (including Richard Dawkins and Ta-Nehisi Coates), and NYPL patrons going about their business in the library’s branches all over the city. The result is almost hypnotic and, perhaps surprisingly, deeply moving. It makes a case for having faith in the public institutions where ordinary people work — away from the limelight, without trying to score political points — in order to make our communities truly better.

                  Ex Libris will air on PBS in the fall and then be available to cardholders in many library systems across the country via Kanopy.

                  1) Lady Bird

                  Lady Bird topped my list almost instantly, and only rose in my estimation on repeated viewings. For many who saw it (including me), it felt like a movie made not just for but about me. Lady Bird is a masterful, exquisite coming-of-age comedy starring the great Saoirse Ronan as Christine — or “Lady Bird,” as she’s re-christened herself — and it’s as funny, smart, and filled with yearning as its heroine. Writer-director Greta Gerwig made the film as an act of love, not just toward her hometown of Sacramento but also toward girlhood, and toward the feeling of always being on the outside of wherever real life is happening. Lady Bird is the rare movie that manages to be affectionate, entertaining, hilarious, witty, and confident. And one line from it struck me as the guiding principle of many of the year’s best films: “Don’t you think they are the same thing? Love, and attention?”

                  Lady Bird is currently streaming on Amazon Prime and available to digitally rent on Amazon, Google Play, and YouTube.

                  Honorable mentions: Marjorie Prime, Phantom Thread, Casting JonBenet, The Post, The Shape of Water, Logan Lucky, I, Tonya, The Lost City of Z, Graduation, Spettacolo, Loveless, Restless Creature: Wendy Whelan, In Transit, The Reagan Show

                  ================================================ FILE: test/test-pages/videos-1/source.html ================================================ How to watch the 21 best films of 2017 - Vox clock menu more-arrow no yes

                  The 21 best movies of 2017

                  How to watch the greatest movies of the year, from Lady Bird and Dunkirk to Get Out and The Big Sick.

                  By Alissa Wilkinson Updated
                  Javier Zarracina/Vox

                  In the introduction to her review anthology For Keeps: 30 Years at the Movies, the legendary film critic Pauline Kael wrote, “I’m frequently asked why I don’t write my memoirs. I think I have.” She meant what most movie critics realize at some point: that reading your past reviews and revisiting the lists of films you liked most during the year reveals not just something about a particular year in cinema, but something about you as well.

                  That’s the feeling I get constructing my list of the best films of 2017, a year that overflowed with great films in every genre, from horror and romantic comedy to documentary and arthouse drama. Some of the films on my list have commonalities — ghosts, meditations on memory and interpersonal connection, and women who refuse to behave — but mostly they underscore just how vibrant cinema remains as an art form, even in the midst of massive cultural shifts in the industry and beyond. And it is a keen reminder to me of all the 2017 conversations I’ve had around and at the movies — and the ways I will never be the same.

                  Here are my top 21 films of 2017 and how to watch them at home, with 14 honorable mentions.

                  21) Star Wars: The Last Jedi

                  I am as shocked as anyone that a Star Wars movie found its way onto my list — but I was bowled over by The Last Jedi, which may be one of the series’ best. In the hands of writer-director Rian Johnson (who will also oversee a new Star Wars trilogy), The Last Jedi is beautiful to look at and keeps its eye on the relationships between characters and how they communicate with one another, in addition to the bigger galactic story. The same characters are back, but they seem infused with new life, and the galaxy with a new kind of hope. The movie’s best details are in the strong bonds that develop between characters, and I left the film with the realization that for the first time in my life, I loved a Star Wars movie. Now I understand the magic.

                  Star Wars: The Last Jedi is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  20) Faces Places

                  The unusual documentary Faces Places (in French, Visages Villages) turns on the friendship between the accomplished street artist JR and legendary film director Agnès Varda, whose work was central to the development of the French New Wave movement. The pair (whose difference in age is 55 years) met after years of admiring each other’s work and decided to create a documentary portrait of France — by making a number of actual portraits. The film chronicles a leg of the "Inside Outside Project," a roving art initiative in which JR makes enormous portraits of people he meets and pastes them onto buildings and walls. In the film, Varda joins him, and as they talk to people around the country, they grow in their understanding of themselves and of each other. The development of their friendship, which is both affectionate and mutually sharpening, forms Faces Places’ emotional center.

                  Faces Places is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  19) Ingrid Goes West

                  Ingrid Goes West is a twisted and dark comedy — part addiction narrative, part stalker story — and yet it’s set in a world that’s almost pathologically cheery: the glossy, sunny, nourishing, superfood- and superlative-loving universe of Instagram celebrity. But despite Ingrid Goes West’s spot-on take on that world, the best thing about the film is that it refuses to traffic in lazy buzzwords and easy skewering, particularly at the expense of young women. Instead, the movie conveys that behind every Instagram image and meltdown is a real person, with real insecurities, real feelings, and real problems. And it recognizes that living a life performed in public can be its own kind of self-deluding prison.

                  Ingrid Goes West is currently streaming on Hulu and available to digitally rent on YouTube and Google Play.

                  18) Lady Macbeth

                  Lady Macbeth is no placid costume drama. Adapted from an 1865 Russian novella by Nikolai Leskov, the movie follows Katherine (the astounding Florence Pugh), a woman in the Lady Macbeth line characterized by a potent cocktail of very few scruples and a lot of determination. She's a chilling avatar for the ways that class and privilege — both obvious and hidden — insulate some people from the consequences of their actions while damning others. Lady Macbeth is also a dazzling directorial debut from William Oldroyd, a thrilling combination of sex, murder, intrigue, and power plays. It’s visually stunning, each frame composed so carefully and deliberately that the wildness and danger roiling just below the surface feels even more frightening. Each scene ratchets up the tension to an explosive, chilling end.

                  Lady Macbeth is currently streaming on HBO Go and HBO Now, and it is available to digitally rent on Amazon Prime, Vudu, YouTube, iTunes, and Google Play.

                  17) BPM (Beats Per Minute)

                  BPM (Beats Per Minute) is a remarkably tender and stirring story of the Paris chapter of ACT UP, an AIDS activism group, and the young people who found themselves caught in the crosshairs of the AIDS crisis in the early 1990s. The film follows both the group's actions and the individual members’ shifting relationships to one another — enemies becoming friends, friends becoming lovers, lovers becoming caretakers — as well as their struggles with the disease wracking their community. As an account of the period, it’s riveting; as an exploration of life and love set at the urgent intersection of the political and the personal, it’s devastating.

                  BPM (Beats Per Minute) is currently streaming on Hulu and available to digitally rent on Google Play and YouTube.

                  16) The Big Sick

                  Few 2017 movies could top the charm and tenderness of The Big Sick, which hits all the right romantic comedy notes with one unusual distinction: It feels like real life. That’s probably because The Big Sick is written by real-life married couple Emily V. Gordon and Silicon Valley's Kumail Nanjiani, and based on their real-life romance. The Big Sick — which stars Nanjiani as a version of himself, alongside Zoe Kazan as Emily — is funny and sweet while not backing away from matters that romantic comedies don’t usually touch on, like serious illness, struggles in long-term marriages, and religion. As it tells the couple’s story, which takes a serious turn when Emily falls ill with a mysterious infection and her parents (played by Holly Hunter and Ray Romano) come to town, it becomes a funny and wise story about real love.

                  The Big Sick is currently streaming on Amazon Prime and available to digitally rent on iTunes, Vudu, Amazon, YouTube, and Google Play.

                  15) Mother!

                  There’s so much pulsing beneath the surface of Mother! that it’s hard to grab on to just one theme as what it “means.” It’s full-on apocalyptic fiction, and like all stories of apocalypse, it’s intended to draw back the veil on reality and show us what’s really beneath. And this movie gets wild: If its gleeful cracking apart of traditional theologies doesn’t get you (there’s a lot of Catholic folk imagery here, complete with an Ash Wednesday-like mud smearing on the foreheads of the faithful), its bonkers scenes of chaos probably will. Mother! is a movie designed to provoke fury, ecstasy, madness, catharsis, and more than a little awe. Watching it, and then participating in the flurry of arguments and discussions unpacking it, was among my best moviegoing experiences of 2017.

                  Mother! is available to digitally purchase on Google Play and YouTube.

                  14) A Ghost Story

                  Director David Lowery filmed A Ghost Story in secret, then premiered it at the Sundance Film Festival to critical acclaim. The movie starts out being about a grieving widow (Rooney Mara) trying to live through the pain of losing her beloved husband, but it soon shifts focus to the ghost of her husband (Casey Affleck, covered in a sheet), evolving into a compelling rumination on the nature of time, memory, history, and the universe. Bathed in warm humor and wistful longing, it's a film that stays with you long after it’s over, a lingering reminder of the inextricable link between love and place.

                  A Ghost Story is available to digitally rent on iTunes, Vudu, Amazon, Google Play, and YouTube.

                  13) The Square

                  The Square is currently streaming on Hulu and available to digitally rent on Google Play and YouTube.

                  12) Dunkirk

                  Dunkirk, a true cinematic achievement from acclaimed director Christopher Nolan, backs off conventional notions of narrative and chronology as much as possible, while leaning headfirst into everything else that makes a movie a visceral work of art aimed at the senses: the images, the sounds, the scale, the swelling vibrations of it all. You can’t smell the sea spray, but your brain may trick you into thinking you can. Nolan’s camera pushes the edges of the screen as far as it can as Dunkirk engulfs the audience in something that feels like a lot more than a war movie. It’s a symphony for the brave and broken, and it resolves in a major key — but one with an undercurrent of sorrow, and of sober warning. Courage in the face of danger is not just for characters in movies.

                  Dunkirk is currently streaming on HBO Go and HBO Now, and available to digitally rent on Google Play and YouTube.

                  11) Rat Film

                  Rat Film is about rats, yes — and rat poison experts and rat hunters and people who keep rats as pets. But it’s also about the history of eugenics, dubious science, “redlining,” and segregated housing in Baltimore. All these pieces come together to form one big essay, where the meaning of each vignette only becomes clearer in light of the whole. It’s a fast-paced, no-holds-barred exploration of a damning history, and it accrues meaning as the images, sounds, and text pile up.

                  Rat Film is available to digitally rent on YouTube and Google Play.

                  10) A Quiet Passion

                  A Quiet Passion is technically a biographical film about Emily Dickinson, but it transcends its genre to become something more like poetry. It’s a perplexing and challenging film, crafted without the traditional guardrails that guide most biographical movies — dates, times, major accomplishments, and so on. Time slips away in the film almost imperceptibly, and the narrative arc doesn’t yield easily to the viewer. Cynthia Nixon plays Emily Dickinson, whose poetry and life is a perfect match for the signature style of director Terence Davies: rich in detail, deeply enigmatic, and weighed down with a kind of sparkling, joy-tinged sorrow. A Quiet Passion is a portrait, both visual and narrative, of the kind of saint most modern people can understand: one who is certain of her uncertainty, and yearning to walk the path on which her passion and longing meet.

                  A Quiet Passion is currently streaming on Amazon Prime and available to digitally rent or purchase on iTunes, Vudu, Amazon, YouTube, and Google Play.

                  9) Columbus

                  Columbus is a stunner of a debut from video essayist turned director Kogonada. Haley Lu Richardson stars as Casey, a young woman living in Columbus, Indiana, who cares for her mother, works at a library, and harbors a passion for architecture. (Columbus is a mecca for modernist architecture scholars and enthusiasts.) When a visiting architecture scholar falls into a coma in Columbus, his estranged son Jin (John Cho) arrives to wait for him and strikes up a friendship with Casey, who starts to show him her favorite buildings. The two begin to unlock something in each other that’s hard to define but life-changing for both. Columbus is beautiful and subtle, letting us feel how the places we build and the people we let near us move and mold us.

                  Columbus is currently streaming on Hulu and available to rent on Google Play and YouTube.

                  8) The Florida Project

                  Sean Baker’s The Florida Project unfolds at first like a series of sketches about the characters who live in a purple-painted, $35-a-night motel called the Magic Castle down the street from Disney World. The film is held together by the hysterical antics of a kid named Moonee and her pack of young friends, as well as long-suffering hotel manager Bobby (a splendid, warm Willem Dafoe), who tries to put up with it all while keeping some kind of order. But as The Florida Project goes on, a narrative starts to form, one that chronicles with heartbreaking attention the sort of dilemmas that face poor parents and their children in America, and the broken systems that try to cope with impossible situations.

                  The Florida Project is currently streaming on Amazon Prime and available to digitally rent on YouTube, Vudu, and Google Play.

                  7) Call Me by Your Name

                  Luca Guadagnino’s gorgeous film Call Me by Your Name adapts André Aciman’s 2007 novel about a precocious 17-year-old named Elio (Timothée Chalamet), who falls in lust and love with his father’s 24-year-old graduate student Oliver (Armie Hammer). It’s remarkable for how it turns literature into pure cinema, all emotion and image and heady sensation. Set in 1983 in Northern Italy, Call Me by Your Name is less about coming out than coming of age, but it also captures a particular sort of love that’s equal parts passion and torment, a kind of irrational heart fire that opens a gate into something longer-lasting. The film is a lush, heady experience for the body, but it’s also an arousal for the soul.

                  Call Me By Your Name is available to digitally purchase on Amazon, YouTube, and Google Play.

                  6) Personal Shopper

                  In her second collaboration with French director Olivier Assayas, Kristen Stewart plays a personal shopper to a wealthy socialite, with a sideline as an amateur ghost hunter who’s searching for her dead twin brother. Personal Shopper is deeper than it seems at first blush, a meditation on grief and an exploration of “between” places — on the fringes of wealth, and in the space between life and death. Some souls are linked in a way that can’t be shaken, and whether or not there’s an afterlife doesn’t change the fact that we see and sense them everywhere. (Personal Shopper also has one of the most tense extended scenes involving text messaging ever seen onscreen.)

                  Personal Shopper is currently streaming on Showtime and available to rent on Vudu, YouTube, Amazon, iTunes, and Google Play.

                  5) Princess Cyd

                  Stephen Cone is a master of small, carefully realized filmmaking; his earlier films such as The Wise Kids and Henry Gamble’s Birthday Party combine an unusual level of empathy for his characters with an unusual combination of interests: love, desire, sexual awakenings, and religion. Princess Cyd is his most accomplished film yet, about a young woman named Cyd (Jessie Pinnick) who finds herself attracted to Katie (Malic White), a barista, while visiting her Aunt Miranda (Rebecca Spence, playing a character modeled on the author Marilynne Robinson) in Chicago. As she works through her own sexual awakening with Katie, Cyd unwinds some of the ways Miranda’s life has gotten too safe. They provoke each other while forming a bond and being prodded toward a bigger understanding of the world. It is a graceful and honest film, and it feels like a modest miracle.

                  Princess Cyd is currently streaming on Netflix and available to digitally rent on Google Play and YouTube.

                  4) Get Out

                  Racism is sinister, frightening, and deadly. But Get Out (a stunning directorial debut from Key & Peele's Jordan Peele) isn’t about the blatantly, obviously scary kind of racism — burning crosses and lynchings and snarling hate. Instead, it’s interested in showing how the parts of racism that try to be aggressively unscary are just as horrifying, and it’s interested in making us feel that horror in a visceral, bodily way. In the tradition of the best classic social thrillers, Get Out takes a topic that is often approached cerebrally — casual racism — and turns it into something you feel in your tummy. And it does it with a wicked sense of humor.

                  Get Out is currently streaming on HBO Go and HBO Now, and is available to digitally rent on iTunes, Amazon, Google Play, YouTube, and Vudu.

                  3) The Work

                  The Work is an outstanding, astonishing accomplishment and a viewing experience that will leave you shaken (but in a good way). At Folsom Prison in California, incarcerated men regularly participate in group therapy, and each year other men from the “outside” apply to participate in an intense four-day period of group therapy alongside Folsom’s inmates. The Work spends almost all of its time inside the room where that therapy happens, observing the strong, visceral, and sometimes violent emotions the men feel as they expose the hurt and raw nerves that have shaped how they encounter the world. Watching is not always easy, but by letting us peek in, the film invites viewers to become part of the experience — as if we, too, are being asked to let go.

                  The Work is streaming on Topic.com and available to digitally rent on Google Play and YouTube.

                  2) Ex Libris

                  Frederick Wiseman is one of the towering giants of nonfiction film, a keen observer of American institutions — ranging from prisons to dance companies to welfare offices — for the past half-century. Ex Libris is his mesmerizing look at the New York Public Library and the many functions it fills, which go far beyond housing books. Wiseman works in the observational mode, which means his films contain no captions, dates, or talking-head interviews: We just see what his camera captured, which in this case includes community meetings, benefit dinners, after-school programs, readings with authors and scholars (including Richard Dawkins and Ta-Nehisi Coates), and NYPL patrons going about their business in the library’s branches all over the city. The result is almost hypnotic and, perhaps surprisingly, deeply moving. It makes a case for having faith in the public institutions where ordinary people work — away from the limelight, without trying to score political points — in order to make our communities truly better.

                  Ex Libris will air on PBS in the fall and then be available to cardholders in many library systems across the country via Kanopy.

                  1) Lady Bird

                  Lady Bird topped my list almost instantly, and only rose in my estimation on repeated viewings. For many who saw it (including me), it felt like a movie made not just for but about me. Lady Bird is a masterful, exquisite coming-of-age comedy starring the great Saoirse Ronan as Christine — or “Lady Bird,” as she’s re-christened herself — and it’s as funny, smart, and filled with yearning as its heroine. Writer-director Greta Gerwig made the film as an act of love, not just toward her hometown of Sacramento but also toward girlhood, and toward the feeling of always being on the outside of wherever real life is happening. Lady Bird is the rare movie that manages to be affectionate, entertaining, hilarious, witty, and confident. And one line from it struck me as the guiding principle of many of the year’s best films: “Don’t you think they are the same thing? Love, and attention?”

                  Lady Bird is currently streaming on Amazon Prime and available to digitally rent on Amazon, Google Play, and YouTube.

                  Honorable mentions: Marjorie Prime, Phantom Thread, Casting JonBenet, The Post, The Shape of Water, Logan Lucky, I, Tonya, The Lost City of Z, Graduation, Spettacolo, Loveless, Restless Creature: Wendy Whelan, In Transit, The Reagan Show

                  Next Up In Culture

                  This Article has a component height of 131. The sidebar size is long.

                  The Latest

                  I have always been a True Detective skeptic. Season 3 made a believer.

                  By Todd VanDerWerff

                  Undocumented immigrants pay millions of dollars in state taxes — even in the reddest states

                  By Alexia Fernández Campbell

                  Meet the first Democrat running for president on climate change

                  By David Roberts

                  The colorful design trend aiming to soothe these anxious times

                  By Daisy Alioto

                  Jay Inslee just launched a presidential campaign that will be all about climate change

                  By Ella Nilsen

                  Vox Sentences: Trump gets nothing

                  By Nicole Fallert
                  ================================================ FILE: test/test-pages/videos-2/expected-metadata.json ================================================ { "title": "Screenshot : «Vape Wave», «6 Days», «Alphonse Président»…", "byline": "Alexandre Hervaud, Jérémy Piette", "dir": null, "lang": "fr", "excerpt": "Séries, documentaires, programmes jeunesse… Retrouvez les recommandations de Libération pour savoir quoi regarder sur vos écrans cette semaine.\nPour dépasser...", "siteName": "Libération", "publishedTime": "2017-11-24T18:42:20.314667", "readerable": true } ================================================ FILE: test/test-pages/videos-2/expected.html ================================================

                  Séries, documentaires, programmes jeunesse… Retrouvez les recommandations de Libération pour savoir quoi regarder sur vos écrans cette semaine.

                  Pour dépasser le tabac

                  Vape Wave (documentaire, 1h28, Planète+)

                  Pendant quelques jours, le doute a plané : l’Etat comptait-il vraiment légiférer contre la cigarette dans les films français, que ce soit via une interdiction pure et simple ou via un système de «punition» (coupe des aides CNC, par exemple) pour les longs-métrages qui sentent le mégot ? Si le rétropédalage de la ministre Buzyn n’en est pas vraiment un (elle n’avait jamais clairement menacé le septième art), la polémique a le mérite de pointer la (sur)représentation clopesque sur écran. Et si, comme c’est le cas dans la vie quotidienne, on voyait progressivement les cigarettes électroniques remplacer les tiges nicotinées authentiques ? Que ceux qui mettraient en doute le potentiel cinématographique des vapoteuses se ruent sur Vape Wave, documentaire militant signé Jan Kounen, ex-fumeur reconverti à la vape dont les images magnifient les volutes de vapeur recrachée.

                  Si le film du réalisateur de Dobermann et 99 Francs part un peu dans tous les sens, il a le mérite de défendre avec une passion contagieuse ce qui semble, de loin, être le meilleur et plus sain substitut à la clope, n’en déplaise aux mesures restrictives imposées en France à son égard. Financé en partie via crowdfunding, le documentaire a été présenté par Kounen à travers toute la France lors de projection tenant quasiment de l’évangélisation. Disponible en VOD/DVD, il a été diffusé cette semaine sur la chaîne Planète+, qui le rediffusera les 25/11, 30/11 et 02/12 prochains. (Alexandre Hervaud)

                  Pour écouter parler un génie

                  Dans la tête d’Alan Moore (websérie documentaire, 8x5min, Arte Creative)

                  Le week-end dernier, Libération publiait un portrait de der consacré à l’auteur britannique Alan Moore, connu pour ses BD cultes (V pour Vendetta, Watchmen, From Hell), à l’occasion de la sortie de son deuxième roman, le pavé Jérusalem. En attendant l’imminente sortie d’une version longue de son entretien avec Libé, on pourra se replonger dans les épisodes d’une websérie documentaire d’Arte Creative en 8 épisodes consacré au maître. Brexit, magie, Anonymous font partie des sujets discutés avec le maître au fil de ce programme sobrement intitulé Dans la tête d’Alan Moore. (A.H.)

                  Pour honorer la mémoire d’une icône queer

                  The Death and Life of Marsha P. Johnson (docu, 1h45, Netflix)

                  Marsha, la «Rosa Parks du mouvement LGBTQ». Marsha «la prostituée, l’actrice et la sainte, modèle d’Andy Warhol» ou encore Marsha l’élaborée, la radicale, «avec ses plumes et ce maquillage qu’elle ne mettait jamais bien». «Queen Marsha» a été retrouvée morte dans l’Hudson en juillet 1992, alors qu’on la voyait encore parader dans les rues de Greenwich Village quelques jours auparavant. Un choc glaçant. Là où son corps a été repêché puis ingratement déposé, les sans-abri ont constitué le lendemain un mémorial de bouteilles et de plantes qui délimitent les contours de l’absente.

                  Marsha P. Johnson de son nom complet, icône queer, femme transgenre noire américaine et emblème de la lutte pour les droits des LGBTQ avait été l’une des premières à s’engager lors des émeutes de Stonewall à New York, en 1969 : «C’est la révolution. Dieu merci.» Marsha était une fleur souriante au parfum d’espoir. Le documentaire The Death and Life of Marsha P. Johnson du cinéaste David France relate l’enquête de l’activiste Victoria Cruz, membre de l’organisation Anti-Violence Project à New York qui, avant de prendre sa retraite, réclame que lumière soit faite sur la disparition de l’icône […] Lire la suite de la critique de Jérémy Piette sur Libération.fr

                  Pour Michel Vuilermoz (et rien d’autre)

                  Alphonse President (série, 10x26, OCS Max)

                  Un temps baptisée French Touch, la série Alphonse Président est le dernier né des programmes originaux made in OCS. On savait les budgets de la chaîne bien moins généreux que ceux de Canal+ (voire que ceux de France 3 Limousin), et cette série le prouve à nouveau régulièrement, notamment lors d’une scène de conférence de presse alternant plans larges d’une authentique conf' à l’Elysée période François Hollande et plans serrés d’acteurs filmés dans un château des Pays de la Loire où a eu lieu le tournage. Le principal atout (et quel atout) de cette série écrite et réalisée par Nicolas Castro (Des lendemains qui chantent, 2014) réside dans son interprète principal, Michel Vuillermoz.

                  Dans le rôle d’un sénateur ringard devenu par un concours de circonstances président de la République, ce pensionnaire de la Comédie-Française et complice d’Albert Dupontel fait des merveilles, notamment lorsque le scénario lui prête des répliques enflammées typiques de la langue de bois politicienne – pas étonnant qu’il brasse du vent, son personnage de prof d’histoire retraité s’appelle Alphonse Dumoulin. C’est lorsqu’il n’est plus à l’écran que les choses se gâtent : si Jean-Michel Lahmi (de la bande d’Edouard Baer) fait le job en grand patron des flics, difficile de croire une seconde à Nabiha Akkari dans le rôle de la Première ministre – et pas uniquement parce que l’idée d’avoir une femme trentenaire issue de la diversité à Matignon sonne hélas comme un doux rêve en 2017. Si, en matière de fiction politique sérieuse, un Baron Noir n’a pas grand-chose à envier à un House of Cards, côté comique la France est encore loin d’avoir son Veep. Gageons que la génération LREM saura largement inspirer des scénaristes moqueurs. (A.H.)

                  Pour les coulisses d’un tournage dément

                  Jim & Andy (documentaire, 1h33, Netflix) 

                  A la sortie de Man on the Moon (2000), le magnifique film de Milos Forman consacré à Andy Kaufman – comique et génie de la performance absurde mort en 1984 –, le cinéaste et les acteurs insistaient dans chaque interview sur l’in­croyable comportement de Jim Carrey pendant le tournage : il aurait été comme possédé par Kaufman, se prenant pour lui 24 heures sur 24. Certains affirmaient même ne jamais avoir eu l’impression que l’acteur était présent, tant son modèle avait littéralement pris sa place. Nous en avons aujourd’hui la preuve en images car tout cela avait été filmé par Bob Zmuda et Lynne Margulies, l’ancien complice et la veuve de Kaufman.

                  Dans le passionnant Jim & Andy : the Great Beyond, disponible sur Netflix, Chris Smith a monté ces documents inédits parallèlement à un entretien dans lequel Jim Carrey revient sur cette expérience unique. Lire la suite de la critique de Marcos Uzal sur Liberation.fr

                  Pour un trip sibérien en totale autarcie

                  Braguino (documentaire, 50min, Arte)

                  La querelle peut se trouver derrière toutes les portes, y compris celle de l’exil. On a beau croire avoir tourné le dos à tout, à cette inclination humaine à nourrir sa propre haine, l’allergie peut regermer fissa sur une peau qui frissonne à l’approche de ce voisin que l’on ne comprend pas. Issu d’une lignée de vieux-croyants orthodoxes russes, Sacha Braguine a pris sa famille sous le bras, loin de toute autre présence humaine en taïga sibérienne. Un autre groupe, les Kiline, a décidé d’en faire de même et de s’installer de l’autre côté de la rivière. Qui est arrivé en premier ? Qui menace l’autre ? L’histoire de l’impossible communauté peut commencer.

                  La lecture d’Ermites dans la taïga (1992) de Vassili Peskov, authentique récit sur la famille Lykov opérant une migration similaire en 1938, a poussé l’artiste Clément Cogitore à rencontrer les Braguine, puis à se faire témoin de la bisbille de voisinage en 2016. Il en est revenu avec un nouveau film d’une cinquantaine de minutes : Braguino, soutenu par le prix Le Bal de la jeune création avec l’ADAGP. Le documentaire y frôle son déguisement fictionnel, tant ce qui s’y déroule convoque une dramaturgie comme invoquée par on ne sait quel rituel vaudou […] Lire la suite de la critique de Jérémy Piette sur Liberation.fr, le film diffusé cette semaine sur Arte est visible en intégralité ci-dessus.

                  Pour un thriller tiré de faits réels

                  6 Days (film, 1h34, Netflix)

                  Fin avril 1980, l’ambassade d’Iran à Londres a été le théâtre d’une prise d’otages largement médiatisée : une trentaine de personnes ont ainsi été retenues pendant six jours par des soldats iraniens dissidents exigeant la libération de 91 prisonniers. Avec Margaret Thatcher au 10 Downing Street à l’époque, pas question pour l’Angleterre d’avoir l’air mou du genou sur la réponse à apporter à cette crise scrutée par les caméras du monde entier. Le SAS (Special Air Service) est sur le coup : l’opération Nimrod se met en place pour prendre d’assaut l’ambassade.

                  Inspiré par cet épisode, 6 Days de Toa Fraser (The Dead Lands, 2014) est un thriller carré pouvant compter sur l'autorité naturelle de Mark Strong (Kingsman) ici recyclé en flic londonien et sur la néo-badass attitude de Jamie Bell, bien loin du freluquet danseur de Billy Elliot puisqu'on le retrouve ici en soldat chargé d’organiser l’opération de secours. Attention, la bande-annonce ci-dessus dévoile à peu près l’intégralité des scènes d’action du film. (A.H.)

                  Alexandre Hervaud , Jérémy Piette

                  ================================================ FILE: test/test-pages/videos-2/source.html ================================================ Screenshot : «Vape Wave», «6 Days», «Alphonse Président»… - Culture / Next Search Direct User 100 Zoom Quiz ? Libération Diamond clock xml netvibes live anciens-numeros data desintox diapo edito election-2017 election essentiel generique idee jo next portrait radio son alerte top-100 star une video scroll politiques food sciences Facebook Whatsapp Twitter insta vine later glass Mail print Facebook Instagram Twitter Calendar download cross zoom-in zoom-out previous next truck visa mastercard user-libe user-doc user-doc-list user-mail user-security user-settings user-shop user-star Ruban abo Losange orange List check Most read Ptit Libé sport blog voyage

                  «C'est important que les agricultures alternatives soient visibles au Salon»

                  1 mars 2019 à 15:20

                  Immobilier : à Paris, le mètre carré frôle la barre des 10 000 euros

                  1 mars 2019 à 15:15

                  Air France-KLM : Paris et La Haye calment le jeu

                  1 mars 2019 à 13:16

                  Ligue du LOL : deux salariés licenciés aux «Inrocks»

                  1 mars 2019 à 10:55

                  Fespaco : des actrices lancent une pétition contre un réalisateur burkinabè

                  1 mars 2019 à 15:17

                  #VonHier, un hashtag où débattre de l'obsession des Allemands pour les origines

                  1 mars 2019 à 10:44

                  Procès Nemmouche : défense et indécence

                  28 février 2019 à 20:36

                  Triple inculpation imminente pour un «roi Bibi» toujours dans le déni

                  28 février 2019 à 20:36

                  Quand le genre et la race divisent le milieu universitaire

                  1 mars 2019 à 12:26

                  «L’anarchisme convivial d’Illich séduit ceux qui veulent vivre leur autonomie face aux institutions»

                  28 février 2019 à 18:06

                  Quand le genre et la race divisent le milieu universitaire

                  28 février 2019 à 17:56

                  Oscars : le black, le green et «Miss Daisy II»

                  28 février 2019 à 17:56

                  De Little Simz à Sakamoto, cinq sons de sortie ce vendredi

                  1 mars 2019 à 06:43

                  Fashion Week : au petit trot

                  1 mars 2019 à 06:14

                  «Eins Zwei Drei», le clown du spectacle

                  28 février 2019 à 18:46

                  Direction d’opéras : prise de têtes à Paris et Lyon

                  28 février 2019 à 18:46

                  Le colonialisme a-t-il été présenté comme une «œuvre civilisatrice» à l'école publique ?

                  1 mars 2019 à 13:38

                  L'Euro a-t-il vraiment fait perdre 56 000€ à chaque Français?

                  1 mars 2019 à 12:54

                  Les animaux exposés au salon de l'Agriculture seront-ils directement envoyés à l'abattoir ?

                  1 mars 2019 à 12:45

                  Si un député est élu au Parlement européen, organise-t-on une nouvelle élection ?

                  28 février 2019 à 17:44
                  Abonnement
                  100% numérique
                  8€ par mois sans engagement                   Formule Intégrale
                  Libération en version papier et numérique
                  Èvénements
                  La Boutique
                  Unes en affiches
                  Les unes cultes en tirage photo                   Relire Libé
                  Commander des anciens numéros
                    info /
                    Sur vos écrans cette semaine

                    Screenshot : «Vape Wave», «6 Days», «Alphonse Président»…

                    Par Alexandre Hervaud et Jérémy Piette —
                    «Vape Wave», «6 Days», «Alphonse président» et «Braguino»
                    «Vape Wave», «6 Days», «Alphonse président» et «Braguino» DR

                    Séries, documentaires, programmes jeunesse… Retrouvez les recommandations de Libération pour savoir quoi regarder sur vos écrans cette semaine.

                    Pour dépasser le tabac

                    Vape Wave (documentaire, 1h28, Planète+)

                    Pendant quelques jours, le doute a plané : l’Etat comptait-il vraiment légiférer contre la cigarette dans les films français, que ce soit via une interdiction pure et simple ou via un système de «punition» (coupe des aides CNC, par exemple) pour les longs-métrages qui sentent le mégot ? Si le rétropédalage de la ministre Buzyn n’en est pas vraiment un (elle n’avait jamais clairement menacé le septième art), la polémique a le mérite de pointer la (sur)représentation clopesque sur écran. Et si, comme c’est le cas dans la vie quotidienne, on voyait progressivement les cigarettes électroniques remplacer les tiges nicotinées authentiques ? Que ceux qui mettraient en doute le potentiel cinématographique des vapoteuses se ruent sur Vape Wave, documentaire militant signé Jan Kounen, ex-fumeur reconverti à la vape dont les images magnifient les volutes de vapeur recrachée.

                    Si le film du réalisateur de Dobermann et 99 Francs part un peu dans tous les sens, il a le mérite de défendre avec une passion contagieuse ce qui semble, de loin, être le meilleur et plus sain substitut à la clope, n’en déplaise aux mesures restrictives imposées en France à son égard. Financé en partie via crowdfunding, le documentaire a été présenté par Kounen à travers toute la France lors de projection tenant quasiment de l’évangélisation. Disponible en VOD/DVD, il a été diffusé cette semaine sur la chaîne Planète+, qui le rediffusera les 25/11, 30/11 et 02/12 prochains. (Alexandre Hervaud)

                    Pour écouter parler un génie

                    Dans la tête d’Alan Moore (websérie documentaire, 8x5min, Arte Creative)

                    Le week-end dernier, Libération publiait un portrait de der consacré à l’auteur britannique Alan Moore, connu pour ses BD cultes (V pour Vendetta, Watchmen, From Hell), à l’occasion de la sortie de son deuxième roman, le pavé Jérusalem. En attendant l’imminente sortie d’une version longue de son entretien avec Libé, on pourra se replonger dans les épisodes d’une websérie documentaire d’Arte Creative en 8 épisodes consacré au maître. Brexit, magie, Anonymous font partie des sujets discutés avec le maître au fil de ce programme sobrement intitulé Dans la tête d’Alan Moore. (A.H.)

                    Pour honorer la mémoire d’une icône queer

                    The Death and Life of Marsha P. Johnson (docu, 1h45, Netflix)

                    Marsha, la «Rosa Parks du mouvement LGBTQ». Marsha «la prostituée, l’actrice et la sainte, modèle d’Andy Warhol» ou encore Marsha l’élaborée, la radicale, «avec ses plumes et ce maquillage qu’elle ne mettait jamais bien». «Queen Marsha» a été retrouvée morte dans l’Hudson en juillet 1992, alors qu’on la voyait encore parader dans les rues de Greenwich Village quelques jours auparavant. Un choc glaçant. Là où son corps a été repêché puis ingratement déposé, les sans-abri ont constitué le lendemain un mémorial de bouteilles et de plantes qui délimitent les contours de l’absente.

                    Marsha P. Johnson de son nom complet, icône queer, femme transgenre noire américaine et emblème de la lutte pour les droits des LGBTQ avait été l’une des premières à s’engager lors des émeutes de Stonewall à New York, en 1969 : «C’est la révolution. Dieu merci.» Marsha était une fleur souriante au parfum d’espoir. Le documentaire The Death and Life of Marsha P. Johnson du cinéaste David France relate l’enquête de l’activiste Victoria Cruz, membre de l’organisation Anti-Violence Project à New York qui, avant de prendre sa retraite, réclame que lumière soit faite sur la disparition de l’icône […] Lire la suite de la critique de Jérémy Piette sur Libération.fr

                    Pour Michel Vuilermoz (et rien d’autre)

                    Alphonse President (série, 10x26, OCS Max)

                    Un temps baptisée French Touch, la série Alphonse Président est le dernier né des programmes originaux made in OCS. On savait les budgets de la chaîne bien moins généreux que ceux de Canal+ (voire que ceux de France 3 Limousin), et cette série le prouve à nouveau régulièrement, notamment lors d’une scène de conférence de presse alternant plans larges d’une authentique conf' à l’Elysée période François Hollande et plans serrés d’acteurs filmés dans un château des Pays de la Loire où a eu lieu le tournage. Le principal atout (et quel atout) de cette série écrite et réalisée par Nicolas Castro (Des lendemains qui chantent, 2014) réside dans son interprète principal, Michel Vuillermoz.

                    Dans le rôle d’un sénateur ringard devenu par un concours de circonstances président de la République, ce pensionnaire de la Comédie-Française et complice d’Albert Dupontel fait des merveilles, notamment lorsque le scénario lui prête des répliques enflammées typiques de la langue de bois politicienne – pas étonnant qu’il brasse du vent, son personnage de prof d’histoire retraité s’appelle Alphonse Dumoulin. C’est lorsqu’il n’est plus à l’écran que les choses se gâtent : si Jean-Michel Lahmi (de la bande d’Edouard Baer) fait le job en grand patron des flics, difficile de croire une seconde à Nabiha Akkari dans le rôle de la Première ministre – et pas uniquement parce que l’idée d’avoir une femme trentenaire issue de la diversité à Matignon sonne hélas comme un doux rêve en 2017. Si, en matière de fiction politique sérieuse, un Baron Noir n’a pas grand-chose à envier à un House of Cards, côté comique la France est encore loin d’avoir son Veep. Gageons que la génération LREM saura largement inspirer des scénaristes moqueurs. (A.H.)

                    Pour les coulisses d’un tournage dément

                    Jim & Andy (documentaire, 1h33, Netflix) 

                    A la sortie de Man on the Moon (2000), le magnifique film de Milos Forman consacré à Andy Kaufman – comique et génie de la performance absurde mort en 1984 –, le cinéaste et les acteurs insistaient dans chaque interview sur l’in­croyable comportement de Jim Carrey pendant le tournage : il aurait été comme possédé par Kaufman, se prenant pour lui 24 heures sur 24. Certains affirmaient même ne jamais avoir eu l’impression que l’acteur était présent, tant son modèle avait littéralement pris sa place. Nous en avons aujourd’hui la preuve en images car tout cela avait été filmé par Bob Zmuda et Lynne Margulies, l’ancien complice et la veuve de Kaufman.

                    Dans le passionnant Jim & Andy : the Great Beyond, disponible sur Netflix, Chris Smith a monté ces documents inédits parallèlement à un entretien dans lequel Jim Carrey revient sur cette expérience unique. Lire la suite de la critique de Marcos Uzal sur Liberation.fr

                    Pour un trip sibérien en totale autarcie

                    Braguino (documentaire, 50min, Arte)

                    La querelle peut se trouver derrière toutes les portes, y compris celle de l’exil. On a beau croire avoir tourné le dos à tout, à cette inclination humaine à nourrir sa propre haine, l’allergie peut regermer fissa sur une peau qui frissonne à l’approche de ce voisin que l’on ne comprend pas. Issu d’une lignée de vieux-croyants orthodoxes russes, Sacha Braguine a pris sa famille sous le bras, loin de toute autre présence humaine en taïga sibérienne. Un autre groupe, les Kiline, a décidé d’en faire de même et de s’installer de l’autre côté de la rivière. Qui est arrivé en premier ? Qui menace l’autre ? L’histoire de l’impossible communauté peut commencer.

                    La lecture d’Ermites dans la taïga (1992) de Vassili Peskov, authentique récit sur la famille Lykov opérant une migration similaire en 1938, a poussé l’artiste Clément Cogitore à rencontrer les Braguine, puis à se faire témoin de la bisbille de voisinage en 2016. Il en est revenu avec un nouveau film d’une cinquantaine de minutes : Braguino, soutenu par le prix Le Bal de la jeune création avec l’ADAGP. Le documentaire y frôle son déguisement fictionnel, tant ce qui s’y déroule convoque une dramaturgie comme invoquée par on ne sait quel rituel vaudou […] Lire la suite de la critique de Jérémy Piette sur Liberation.fr, le film diffusé cette semaine sur Arte est visible en intégralité ci-dessus.

                    Pour un thriller tiré de faits réels

                    6 Days (film, 1h34, Netflix)

                    Fin avril 1980, l’ambassade d’Iran à Londres a été le théâtre d’une prise d’otages largement médiatisée : une trentaine de personnes ont ainsi été retenues pendant six jours par des soldats iraniens dissidents exigeant la libération de 91 prisonniers. Avec Margaret Thatcher au 10 Downing Street à l’époque, pas question pour l’Angleterre d’avoir l’air mou du genou sur la réponse à apporter à cette crise scrutée par les caméras du monde entier. Le SAS (Special Air Service) est sur le coup : l’opération Nimrod se met en place pour prendre d’assaut l’ambassade.

                    Inspiré par cet épisode, 6 Days de Toa Fraser (The Dead Lands, 2014) est un thriller carré pouvant compter sur l'autorité naturelle de Mark Strong (Kingsman) ici recyclé en flic londonien et sur la néo-badass attitude de Jamie Bell, bien loin du freluquet danseur de Billy Elliot puisqu'on le retrouve ici en soldat chargé d’organiser l’opération de secours. Attention, la bande-annonce ci-dessus dévoile à peu près l’intégralité des scènes d’action du film. (A.H.)

                    Alexandre Hervaud , Jérémy Piette
                    Un mot à ajouter ?
                    ================================================ FILE: test/test-pages/visibility-hidden/expected-metadata.json ================================================ { "title": "Visibility hidden test", "byline": null, "dir": null, "excerpt": "Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/visibility-hidden/expected.html ================================================

                    Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

                    Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

                    ================================================ FILE: test/test-pages/visibility-hidden/source.html ================================================ Visibility hidden test

                    Lorem

                    Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

                    Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

                    Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

                    Foo

                    Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

                    Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

                    ================================================ FILE: test/test-pages/wapo-1/expected-metadata.json ================================================ { "title": "Attack stokes instability fears in North Africa", "byline": "By Erin Cunningham", "dir": null, "excerpt": "The assault on Tunisia’s most renowned museum, in which gunmen killed at least 19 people, could heighten tensions in a nation that has become deeply divided between pro- and anti-Islamist factions.", "siteName": "Washington Post", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/wapo-1/expected.html ================================================

                    CAIRO — Gunmen opened fire on visitors at Tunisia’s most renowned museum on Wednesday, killing at least 19 people, including 17 foreigners, in an assault that threatened to upset the fragile stability of a country seen as the lone success of the Arab Spring.

                    It was the most deadly terrorist attack in the North African nation in more than a decade. Although no group claimed responsibility, the bloodshed raised fears that militants linked to the Islamic State were expanding their operations.

                    The attackers, clad in military uniforms, stormed the Bardo National Museum on Wednesday afternoon, seizing and gunning down foreign tourists before security forces raided the building to end the siege. The museum is a major tourist draw and is near the heavily guarded national parliament in downtown Tunis.

                    Tunisian Prime Minister Habib Essid said that in addition to the slain foreigners — from Italy, Poland, Germany and Spain — a local museum worker and a security official were killed. Two gunmen died, and three others may have escaped, officials said. About 50 other people were wounded, according to local news reports.

                    “Our nation is in danger,” Essid declared in a televised address Wednesday evening. He vowed that the country would be “merciless” in defending itself.

                    [Read: Why Tunisia, Arab Spring’s sole success story, suffers from Islamist violence]

                    Tunisia, a mostly Muslim nation of about 11 million people, was governed for decades by autocrats who imposed secularism. Its sun-drenched Mediterranean beaches drew thousands of bikini-clad tourists, and its governments promoted education and other rights for women. But the country has grappled with rising Islamist militancy since a popular uprising overthrew its dictator four years ago, setting the stage for the Arab Spring revolts across the region.

                    Thousands of Tunisians have flocked to join jihadist groups in Syria, including the Islamic State, making the country one of the major sources of foreign fighters in the conflict. Tunisian security forces have also fought increasing gunbattles with jihadists at home.

                    Despite this, the country has been hailed as a model of democratic transition as other governments that came to power after the Arab Spring collapsed, often in bloody confrontations. But the attack Wednesday — on a national landmark that showcases Tunisia’s rich heritage — could heighten tensions in a nation that has become deeply divided between pro- and anti-Islamist political factions.

                    Many Tunisians accuse the country’s political Islamists, who held power from 2011 to 2013, of having been slow to respond to the growing danger of terrorism. Islamist politicians have acknowledged that they did not realize the threat that would develop when radical Muslims, who had been repressed under authoritarian regimes, won the freedom to preach freely in mosques.

                    In Washington, White House press secretary Josh Earnest condemned the attack and said the U.S. government was willing to assist Tunisian authorities in the investigation.

                    Gunmen in military uniforms stormed Tunisia's national museum, killing at least 19 people, most of them foreign tourists. (Reuters)

                    “This attack today is meant to threaten authorities, to frighten tourists and to negatively affect the economy,” said Lotfi Azzouz, Tunisia country director for Amnesty International, a London-based rights group.

                    Tourism is critical to Tunisia’s economy, accounting for 15 percent of its gross domestic product in 2013, according to the World Travel and Tourism Council, an industry body. The Bardo museum hosts one of the world’s most outstanding collections of Roman mosaics and is popular with tourists and Tunisians alike.

                    [Bardo museum houses amazing Roman treasures]

                    The attack is “also aimed at the country’s security and stability during the transition period,” Azzouz said. “And it could have political repercussions — like the curtailing of human rights, or even less government transparency if there’s fear of further attacks.”

                    The attack raised concerns that the government, led by secularists, would be pressured to stage a wider crackdown on Islamists of all stripes. Lawmakers are drafting an anti-terrorism bill to give security forces additional tools to fight militants.

                    [Read: Tunisia sends most foreign fighters to Islamic State in Syria]

                    “We must pay attention to what is written” in that law, Azzouz said. “There is worry the government will use the attack to justify some draconian measures.”

                    Tunisian Islamists and secular forces have worked together — often reluctantly — to defuse the country’s political crises in the years since the revolt.

                    Last fall, Tunisians elected a secular-minded president and parliament dominated by liberal forces after souring on Islamist-led rule. In 2011, voters had elected a government led by the Ennahda party — a movement similar to Egypt’s Islamist Muslim Brotherhood. But a political stalemate developed as the party and others tried to draft the country’s new constitution. The Islamists failed to improve a slumping economy. And Ennahda came under fire for what many Tunisians saw as a failure to crack down on Islamist extremists.

                    Map: Flow of foreign fighters to Syria

                    After the collapse of the authoritarian system in 2011, hard-line Muslims known as Salafists attacked bars and art galleries. Then, in 2012, hundreds of Islamists assaulted the U.S. Embassy in Tunis, shattering windows and hurling gasoline bombs, after the release of a crude online video about the prophet Muhammad. The government outlawed the group behind the attack — Ansar al-Sharia, an al-Qaeda-linked organization — and began a crackdown. But the killing of two leftist politicians in 2013 prompted a fresh political crisis, and Ennahda stepped down, replaced by a technocratic government.

                    Tunisia’s current coalition government includes an Ennahda minister in the cabinet. Still, many leftist figures openly oppose collaboration with the movement’s leaders.

                    “Ennahda is responsible for the current deterioration of the situation, because they were careless with the extremists” while they were in power, Azzouz said.

                    The leader of Ennahda, Rachid Ghannouchi, condemned Wednesday’s attack, saying in a statement that it “will not break our people’s will and will not undermine our revolution and our democracy.”

                    Security officials are particularly concerned by the collapse of Libya, where various armed groups are vying for influence and jihadist militants have entrenched themselves in major cities. Tunisians worry that extremists can easily get arms and training in the neighboring country.

                    In January, Libyan militants loyal to the Islamic State beheaded 21 Christians — 20 of them Egyptian Copts — along the country’s coast. They later seized the Libyan city of Sirte.

                    Officials are worried about the number of Tunisian militants who may have joined the jihadists in Libya — with the goal of returning home to fight the Tunis government.

                    Ajmi Lourimi, a member of Ennahda’s general secretariat, said he believed the attack would unite Tunisians in the face of terrorism.

                    “There is a consensus here that this [attack] is alien to our culture, to our way of life. We want to unify against this danger,” Lourimi said. He said he did not expect a wider government campaign against Islamists.

                    “We have nothing to fear,” he said of himself and fellow Ennahda members. “We believe the Interior Ministry should be trained and equipped to fight and counter this militancy.”

                    The last major attack on a civilian target in Tunisia was in 2002, when al-Qaeda militants killed more than 20 people in a car bombing outside a synagogue in the city of Djerba.

                    Heba Habib contributed to this report.

                    Read more:

                    Tunisia’s Islamists get a sobering lesson in governing

                    Tunisia sends most foreign fighters to Islamic State in Syria

                    Tunisia’s Bardo museum is home to amazing Roman treasures

                    ================================================ FILE: test/test-pages/wapo-1/source.html ================================================ Attack stokes instability fears in North Africa - The Washington Post
                    Middle East

                    Attack stokes instability fears in North Africa

                    By Erin Cunningham March 18 at 3:57 PM

                    CAIRO — Gunmen opened fire on visitors at Tunisia’s most renowned museum on Wednesday, killing at least 19 people, including 17 foreigners, in an assault that threatened to upset the fragile stability of a country seen as the lone success of the Arab Spring.

                    It was the most deadly terrorist attack in the North African nation in more than a decade. Although no group claimed responsibility, the bloodshed raised fears that militants linked to the Islamic State were expanding their operations.

                    The attackers, clad in military uniforms, stormed the Bardo National Museum on Wednesday afternoon, seizing and gunning down foreign tourists before security forces raided the building to end the siege. The museum is a major tourist draw and is near the heavily guarded national parliament in downtown Tunis.

                    Tunisian Prime Minister Habib Essid said that in addition to the slain foreigners — from Italy, Poland, Germany and Spain — a local museum worker and a security official were killed. Two gunmen died, and three others may have escaped, officials said. About 50 other people were wounded, according to local news reports.

                    “Our nation is in danger,” Essid declared in a televised address Wednesday evening. He vowed that the country would be “merciless” in defending itself.

                    [Read: Why Tunisia, Arab Spring’s sole success story, suffers from Islamist violence]

                    Tunisia, a mostly Muslim nation of about 11 million people, was governed for decades by autocrats who imposed secularism. Its sun-drenched Mediterranean beaches drew thousands of bikini-clad tourists, and its governments promoted education and other rights for women. But the country has grappled with rising Islamist militancy since a popular uprising overthrew its dictator four years ago, setting the stage for the Arab Spring revolts across the region.

                    Thousands of Tunisians have flocked to join jihadist groups in Syria, including the Islamic State, making the country one of the major sources of foreign fighters in the conflict. Tunisian security forces have also fought increasing gunbattles with jihadists at home.

                    Despite this, the country has been hailed as a model of democratic transition as other governments that came to power after the Arab Spring collapsed, often in bloody confrontations. But the attack Wednesday — on a national landmark that showcases Tunisia’s rich heritage — could heighten tensions in a nation that has become deeply divided between pro- and anti-Islamist political factions.

                    Many Tunisians accuse the country’s political Islamists, who held power from 2011 to 2013, of having been slow to respond to the growing danger of terrorism. Islamist politicians have acknowledged that they did not realize the threat that would develop when radical Muslims, who had been repressed under authoritarian regimes, won the freedom to preach freely in mosques.

                    In Washington, White House press secretary Josh Earnest condemned the attack and said the U.S. government was willing to assist Tunisian authorities in the investigation.

                    Gunmen in military uniforms stormed Tunisia's national museum, killing at least 19 people, most of them foreign tourists. (Reuters)

                    “This attack today is meant to threaten authorities, to frighten tourists and to negatively affect the economy,” said Lotfi Azzouz, Tunisia country director for Amnesty International, a London-based rights group.

                    Tourism is critical to Tunisia’s economy, accounting for 15 percent of its gross domestic product in 2013, according to the World Travel and Tourism Council, an industry body. The Bardo museum hosts one of the world’s most outstanding collections of Roman mosaics and is popular with tourists and Tunisians alike.

                    [Bardo museum houses amazing Roman treasures]

                    The attack is “also aimed at the country’s security and stability during the transition period,” Azzouz said. “And it could have political repercussions — like the curtailing of human rights, or even less government transparency if there’s fear of further attacks.”

                    The attack raised concerns that the government, led by secularists, would be pressured to stage a wider crackdown on Islamists of all stripes. Lawmakers are drafting an anti-terrorism bill to give security forces additional tools to fight militants.

                    [Read: Tunisia sends most foreign fighters to Islamic State in Syria]

                    “We must pay attention to what is written” in that law, Azzouz said. “There is worry the government will use the attack to justify some draconian measures.”

                    Tunisian Islamists and secular forces have worked together — often reluctantly — to defuse the country’s political crises in the years since the revolt.

                    Last fall, Tunisians elected a secular-minded president and parliament dominated by liberal forces after souring on Islamist-led rule. In 2011, voters had elected a government led by the Ennahda party — a movement similar to Egypt’s Islamist Muslim Brotherhood. But a political stalemate developed as the party and others tried to draft the country’s new constitution. The Islamists failed to improve a slumping economy. And Ennahda came under fire for what many Tunisians saw as a failure to crack down on Islamist extremists.

                    Map: Flow of foreign fighters to Syria View Graphic
                    Map: Flow of foreign fighters to Syria

                    After the collapse of the authoritarian system in 2011, hard-line Muslims known as Salafists attacked bars and art galleries. Then, in 2012, hundreds of Islamists assaulted the U.S. Embassy in Tunis, shattering windows and hurling gasoline bombs, after the release of a crude online video about the prophet Muhammad. The government outlawed the group behind the attack — Ansar al-Sharia, an al-Qaeda-linked organization — and began a crackdown. But the killing of two leftist politicians in 2013 prompted a fresh political crisis, and Ennahda stepped down, replaced by a technocratic government.

                    Tunisia’s current coalition government includes an Ennahda minister in the cabinet. Still, many leftist figures openly oppose collaboration with the movement’s leaders.

                    “Ennahda is responsible for the current deterioration of the situation, because they were careless with the extremists” while they were in power, Azzouz said.

                    The leader of Ennahda, Rachid Ghannouchi, condemned Wednesday’s attack, saying in a statement that it “will not break our people’s will and will not undermine our revolution and our democracy.”

                    Security officials are particularly concerned by the collapse of Libya, where various armed groups are vying for influence and jihadist militants have entrenched themselves in major cities. Tunisians worry that extremists can easily get arms and training in the neighboring country.

                    In January, Libyan militants loyal to the Islamic State beheaded 21 Christians — 20 of them Egyptian Copts — along the country’s coast. They later seized the Libyan city of Sirte.


                    Officials are worried about the number of Tunisian militants who may have joined the jihadists in Libya — with the goal of returning home to fight the Tunis government.

                    Ajmi Lourimi, a member of Ennahda’s general secretariat, said he believed the attack would unite Tunisians in the face of terrorism.

                    “There is a consensus here that this [attack] is alien to our culture, to our way of life. We want to unify against this danger,” Lourimi said. He said he did not expect a wider government campaign against Islamists.

                    “We have nothing to fear,” he said of himself and fellow Ennahda members. “We believe the Interior Ministry should be trained and equipped to fight and counter this militancy.”

                    The last major attack on a civilian target in Tunisia was in 2002, when al-Qaeda militants killed more than 20 people in a car bombing outside a synagogue in the city of Djerba.

                    Heba Habib contributed to this report.

                    Read more:

                    Tunisia’s Islamists get a sobering lesson in governing

                    Tunisia sends most foreign fighters to Islamic State in Syria

                    Tunisia’s Bardo museum is home to amazing Roman treasures

                    Erin Cunningham is an Egypt-based correspondent for The Post. She previously covered conflicts in the Middle East and Afghanistan for the Christian Science Monitor, GlobalPost and The National.
                    Comments
                    Show Comments
                    Comments

                    world

                    middle_east

                    Most Read World

                    Our Online Games

                    Play right from this page

                    ================================================ FILE: test/test-pages/wapo-2/expected-metadata.json ================================================ { "title": "Where do strained U.S.-Israeli relations go after Netanyahu’s victory?", "byline": "By Steven Mufson", "dir": null, "excerpt": "Few foreign leaders have so brazenly stood up to President Obama and the relationship could face its next test this month.", "siteName": "Washington Post", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/wapo-2/expected.html ================================================


                    Israeli Prime Minister Benjamin Netanyahu reacts as he visits the Western Wall in Jerusalem on March 18 following his party's victory in Israel's general election. (Thomas Coex/AFP/Getty Images)

                    President Obama told the U.N. General Assembly 18 months ago that he would seek “real breakthroughs on these two issues — Iran’s nuclear program and ­Israeli-Palestinian peace.”

                    But Benjamin Netanyahu’s triumph in Tuesday’s parliamentary elections keeps in place an Israeli prime minister who has declared his intention to resist Obama on both of these fronts, guaranteeing two more years of difficult diplomacy between leaders who barely conceal their personal distaste for each other.

                    The Israeli election results also suggest that most voters there support Netanyahu’s tough stance on U.S.-led negotiations to limit Iran’s nuclear program and his vow on Monday that there would be no independent Palestinian state as long as he is prime minister.

                    “On the way to his election victory, Netanyahu broke a lot of crockery in the relationship,” said Martin Indyk, executive vice president of the Brookings Institution and a former U.S. ambassador to Israel. “It can’t be repaired unless both sides have an interest and desire to do so.”

                    Aside from Russian President Vladi­mir Putin, few foreign leaders so brazenly stand up to Obama and even fewer among longtime allies.

                    Israeli Prime Minister Benjamin Netanyahu pledged to form a new governing coalition quickly after an upset election victory that was built on a shift to the right. (Reuters)

                    In the past, Israeli leaders who risked damaging the country’s most important relationship, that with Washington, tended to pay a price. In 1991, when Prime Minister Yitzhak Shamir opposed the Madrid peace talks, President George H.W. Bush held back loan guarantees to help absorb immigrants from the former Soviet Union. Shamir gave in, but his government soon collapsed.

                    But this time, Netanyahu was not hurt by his personal and substantive conflicts with the U.S. president.

                    “While the United States is loved and beloved in Israel, President Obama is not,” said Robert M. Danin, a senior fellow at the Council on Foreign Relations. “So the perceived enmity didn’t hurt the way it did with Shamir when he ran afoul of Bush in ’91.”

                    Where do U.S.-Israeli relations go from here?

                    In the immediate aftermath of Tuesday’s elections, tensions between the two sides continued to run hot. The Obama administration’s first comments on the Israeli election came with a tough warning about some of the pre-election rhetoric from Netanyahu’s Likud party, which tried to rally right-wing support by saying that Arab Israeli voters were “coming out in droves.”

                    “The United States and this administration is deeply concerned about rhetoric that seeks to marginalize Arab Israeli citizens,” White House press secretary Josh Earnest told reporters aboard Air Force One. “It undermines the values and democratic ideals that have been important to our democracy and an important part of what binds the United States and Israel together.”

                    Earnest added that Netan­yahu’s election-eve disavowal of a two-state solution for Israelis and Palestinians would force the administration to reconsider its approach to peace in the region.

                    Over the longer term, a number of analysts say that Obama and Netan­yahu will seek to play down the friction between them and point to areas of continuing cooperation on military and economic issues.

                    “Both sides are going to want to turn down the rhetoric,” Danin said. “But it is also a structural problem. They have six years of accumulated history. That’s going to put limits on how far they can go together.”

                    The first substantive test could come as early as this month, when the United States hopes that it can finish hammering out the framework of an agreement with Iran.

                    Netanyahu strongly warned against making a “bad deal” during his March 3 address to a joint meeting of Congress, an appearance arranged by Republican congressional leaders and criticized by the Obama administration for making U.S.-Israeli relations partisan on both sides so close to the Israeli election.

                    If a deal is reached and does not pass muster with Netanyahu, he is likely to work with congressional Republicans to try to scuttle the accord.

                    “The Republicans have said they will do what they can to block a deal, and the prime minister has already made clear that he will work with the Republicans against the president,” Indyk said. “That’s where a clash could come, and it’s coming very quickly.”

                    The second test — talks with Palestinians — could be even more difficult. In his September 2013 address to the United Nations, Obama hailed signs of hope.

                    “Already, Israeli and Palestinian leaders have demonstrated a willingness to take significant political risks,” Obama said in his speech. Palestinian Authority President Mahmoud Abbas “has put aside efforts to shortcut the pursuit of peace and come to the negotiating table. Prime Minister Netanyahu has released Palestinian prisoners and reaffirmed his commitment to a Palestinian state.”

                    Today, the signals could not differ more. The Palestinian Authority has said that after it joins the International Criminal Court at The Hague on April 1, it will press war crimes charges against Israel for the bloody Gaza conflict during the summer. Israel, which controls tax receipts, has pledged to punish the Palestinian Authority by freezing its tax revenue.

                    The United States, which gives hundreds of millions of dollars of economic aid to the Palestinian Authority, would be caught in the middle. It has been trying to persuade both sides to stand down, but Netanyahu’s declaration that there would be no Palestinian state on his watch makes that more difficult.

                    “Now it’s hard to see what could persuade the Palestinians” to hold up on their ICC plans, Indyk said. “That has nothing to do with negotiations, but if both sides can’t be persuaded to back down, then they will be on a trajectory that could lead to the collapse of the Palestinian Authority because it can’t pay wages anymore.

                    “That could be an issue forced onto the agenda about the same time as a potential nuclear deal.”

                    Steven Mufson covers the White House. Since joining The Post, he has covered economics, China, foreign policy and energy.

                    ================================================ FILE: test/test-pages/wapo-2/source.html ================================================ Where do strained U.S.-Israeli relations go after Netanyahu’s victory? - The Washington Post
                    Middle East

                    Where do strained U.S.-Israeli relations go after Netanyahu’s victory?


                    Israeli Prime Minister Benjamin Netanyahu reacts as he visits the Western Wall in Jerusalem on March 18 following his party's victory in Israel's general election. (Thomas Coex/AFP/Getty Images)
                    By Steven Mufson March 18 at 12:22 PM

                    President Obama told the U.N. General Assembly 18 months ago that he would seek “real breakthroughs on these two issues — Iran’s nuclear program and ­Israeli-Palestinian peace.”

                    But Benjamin Netanyahu’s triumph in Tuesday’s parliamentary elections keeps in place an Israeli prime minister who has declared his intention to resist Obama on both of these fronts, guaranteeing two more years of difficult diplomacy between leaders who barely conceal their personal distaste for each other.

                    The Israeli election results also suggest that most voters there support Netanyahu’s tough stance on U.S.-led negotiations to limit Iran’s nuclear program and his vow on Monday that there would be no independent Palestinian state as long as he is prime minister.

                    “On the way to his election victory, Netanyahu broke a lot of crockery in the relationship,” said Martin Indyk, executive vice president of the Brookings Institution and a former U.S. ambassador to Israel. “It can’t be repaired unless both sides have an interest and desire to do so.”

                    Aside from Russian President Vladi­mir Putin, few foreign leaders so brazenly stand up to Obama and even fewer among longtime allies.

                    Israeli Prime Minister Benjamin Netanyahu pledged to form a new governing coalition quickly after an upset election victory that was built on a shift to the right. (Reuters)

                    In the past, Israeli leaders who risked damaging the country’s most important relationship, that with Washington, tended to pay a price. In 1991, when Prime Minister Yitzhak Shamir opposed the Madrid peace talks, President George H.W. Bush held back loan guarantees to help absorb immigrants from the former Soviet Union. Shamir gave in, but his government soon collapsed.

                    But this time, Netanyahu was not hurt by his personal and substantive conflicts with the U.S. president.

                    “While the United States is loved and beloved in Israel, President Obama is not,” said Robert M. Danin, a senior fellow at the Council on Foreign Relations. “So the perceived enmity didn’t hurt the way it did with Shamir when he ran afoul of Bush in ’91.”

                    Where do U.S.-Israeli relations go from here?

                    In the immediate aftermath of Tuesday’s elections, tensions between the two sides continued to run hot. The Obama administration’s first comments on the Israeli election came with a tough warning about some of the pre-election rhetoric from Netanyahu’s Likud party, which tried to rally right-wing support by saying that Arab Israeli voters were “coming out in droves.”

                    “The United States and this administration is deeply concerned about rhetoric that seeks to marginalize Arab Israeli citizens,” White House press secretary Josh Earnest told reporters aboard Air Force One. “It undermines the values and democratic ideals that have been important to our democracy and an important part of what binds the United States and Israel together.”

                    Earnest added that Netan­yahu’s election-eve disavowal of a two-state solution for Israelis and Palestinians would force the administration to reconsider its approach to peace in the region.

                    Over the longer term, a number of analysts say that Obama and Netan­yahu will seek to play down the friction between them and point to areas of continuing cooperation on military and economic issues.

                    “Both sides are going to want to turn down the rhetoric,” Danin said. “But it is also a structural problem. They have six years of accumulated history. That’s going to put limits on how far they can go together.”

                    The first substantive test could come as early as this month, when the United States hopes that it can finish hammering out the framework of an agreement with Iran.

                    Netanyahu strongly warned against making a “bad deal” during his March 3 address to a joint meeting of Congress, an appearance arranged by Republican congressional leaders and criticized by the Obama administration for making U.S.-Israeli relations partisan on both sides so close to the Israeli election.

                    If a deal is reached and does not pass muster with Netanyahu, he is likely to work with congressional Republicans to try to scuttle the accord.

                    “The Republicans have said they will do what they can to block a deal, and the prime minister has already made clear that he will work with the Republicans against the president,” Indyk said. “That’s where a clash could come, and it’s coming very quickly.”

                    The second test — talks with Palestinians — could be even more difficult. In his September 2013 address to the United Nations, Obama hailed signs of hope.

                    “Already, Israeli and Palestinian leaders have demonstrated a willingness to take significant political risks,” Obama said in his speech. Palestinian Authority President Mahmoud Abbas “has put aside efforts to shortcut the pursuit of peace and come to the negotiating table. Prime Minister Netanyahu has released Palestinian prisoners and reaffirmed his commitment to a Palestinian state.”

                    Today, the signals could not differ more. The Palestinian Authority has said that after it joins the International Criminal Court at The Hague on April 1, it will press war crimes charges against Israel for the bloody Gaza conflict during the summer. Israel, which controls tax receipts, has pledged to punish the Palestinian Authority by freezing its tax revenue.

                    The United States, which gives hundreds of millions of dollars of economic aid to the Palestinian Authority, would be caught in the middle. It has been trying to persuade both sides to stand down, but Netanyahu’s declaration that there would be no Palestinian state on his watch makes that more difficult.

                    “Now it’s hard to see what could persuade the Palestinians” to hold up on their ICC plans, Indyk said. “That has nothing to do with negotiations, but if both sides can’t be persuaded to back down, then they will be on a trajectory that could lead to the collapse of the Palestinian Authority because it can’t pay wages anymore.

                    “That could be an issue forced onto the agenda about the same time as a potential nuclear deal.”

                    Steven Mufson covers the White House. Since joining The Post, he has covered economics, China, foreign policy and energy.
                    Comments
                    Show Comments
                    Comments

                    world

                    middle_east

                    Most Read Business

                    Our Online Games

                    Play right from this page

                    ================================================ FILE: test/test-pages/webmd-1/expected-metadata.json ================================================ { "title": "Babies Who Eat Peanuts Early May Avoid Allergy", "byline": "By Brenda Goodman, MA\n WebMD Health News", "dir": null, "lang": "en", "excerpt": "Life-threatening peanut allergies have mysteriously been on the rise in the past decade, with little hope for a cure. But a groundbreaking new study may offer a way to stem that rise, while another may offer some hope for those who are already allergic.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/webmd-1/expected.html ================================================

                    Feb. 23, 2015 -- Life-threatening peanut allergies have mysteriously been on the rise in the past decade, with little hope for a cure.

                    But a groundbreaking new study may offer a way to stem that rise, while another may offer some hope for those who are already allergic.

                    Parents have been told for years to avoid giving foods containing peanuts to babies for fear of triggering an allergy. Now research shows the opposite is true: Feeding babies snacks made with peanuts before their first birthday appears to prevent that from happening.

                    The study is published in the New England Journal of Medicine, and it was presented at the annual meeting of the American Academy of Allergy, Asthma and Immunology in Houston. It found that among children at high risk for getting peanut allergies, eating peanut snacks by 11 months of age and continuing to eat them at least three times a week until age 5 cut their chances of becoming allergic by more than 80% compared to kids who avoided peanuts. Those at high risk were already allergic to egg, they had the skin condition eczema, or both.

                    Overall, about 3% of kids who ate peanut butter or peanut snacks before their first birthday got an allergy, compared to about 17% of kids who didn’t eat them.

                    “I think this study is an astounding and groundbreaking study, really,” says Katie Allen, MD, PhD. She's the director of the Center for Food and Allergy Research at the Murdoch Children’s Research Institute in Melbourne, Australia. Allen was not involved in the research.

                    Experts say the research should shift thinking about how kids develop food allergies, and it should change the guidance doctors give to parents.

                    Meanwhile, for children and adults who are already allergic to peanuts, another study presented at the same meeting held out hope of a treatment.

                    A new skin patch called Viaskin allowed people with peanut allergies to eat tiny amounts of peanuts after they wore it for a year.

                    A Change in Guidelines?

                    Allergies to peanuts and other foods are on the rise. In the U.S., more than 2% of people react to peanuts, a 400% increase since 1997. And reactions to peanuts and other tree nuts can be especially severe. Nuts are the main reason people get a life-threatening problem called anaphylaxis.

                    ================================================ FILE: test/test-pages/webmd-1/source.html ================================================ Babies Who Eat Peanuts Early May Avoid Allergy
                    Skip to content

                    Allergies Health Center

                    Tools & Resources

                    Font Size
                    A
                    A
                    A

                    Babies Who Eat Peanuts Early May Avoid Allergy

                    By
                    WebMD Health News
                    Reviewed by Hansa D. Bhargava, MD

                    Feb. 23, 2015 -- Life-threatening peanut allergies have mysteriously been on the rise in the past decade, with little hope for a cure.

                    But a groundbreaking new study may offer a way to stem that rise, while another may offer some hope for those who are already allergic.

                    Parents have been told for years to avoid giving foods containing peanuts to babies for fear of triggering an allergy. Now research shows the opposite is true: Feeding babies snacks made with peanuts before their first birthday appears to prevent that from happening.

                    The study is published in the New England Journal of Medicine, and it was presented at the annual meeting of the American Academy of Allergy, Asthma and Immunology in Houston. It found that among children at high risk for getting peanut allergies, eating peanut snacks by 11 months of age and continuing to eat them at least three times a week until age 5 cut their chances of becoming allergic by more than 80% compared to kids who avoided peanuts. Those at high risk were already allergic to egg, they had the skin condition eczema, or both.

                    Overall, about 3% of kids who ate peanut butter or peanut snacks before their first birthday got an allergy, compared to about 17% of kids who didn’t eat them.

                    “I think this study is an astounding and groundbreaking study, really,” says Katie Allen, MD, PhD. She's the director of the Center for Food and Allergy Research at the Murdoch Children’s Research Institute in Melbourne, Australia. Allen was not involved in the research.

                    Experts say the research should shift thinking about how kids develop food allergies, and it should change the guidance doctors give to parents.

                    Meanwhile, for children and adults who are already allergic to peanuts, another study presented at the same meeting held out hope of a treatment.

                    A new skin patch called Viaskin allowed people with peanut allergies to eat tiny amounts of peanuts after they wore it for a year.

                    A Change in Guidelines?

                    Allergies to peanuts and other foods are on the rise. In the U.S., more than 2% of people react to peanuts, a 400% increase since 1997. And reactions to peanuts and other tree nuts can be especially severe. Nuts are the main reason people get a life-threatening problem called anaphylaxis.

                    1 | 2 | 3 | 4 | 5
                    Next Page >

                    Today on WebMD

                    man blowing nose
                    12 Natural Ways to Defeat Allergies
                    Make these tweaks to your diet, home, and lifestyle.
                    Allergy capsule
                    Need Nasal Allergy Relief?
                    Breathe easier with these products.
                     
                    cat on couch
                    5 Tips to Cut Pet Allergies
                    Live in harmony with your cat or dog.
                    Woman sneezing with tissue in meadow
                    10 Common Allergy Triggers
                    Which ones affect you?
                     

                    blowing nose
                    Article
                    The Truth About Mucus
                    woman with sore throat
                    Article
                    What Is Postnasal Drip?
                     
                    lone star tick
                    Slideshow
                    Identify Bugs and Their Bites
                    Woman blowing nose
                    Slideshow
                    12 Natural Ways to Defeat Allergies
                     

                    Send yourself a link to download the app.

                    Loading ...

                    Please wait...

                    This feature is temporarily unavailable. Please try again later.

                    Thanks!

                    Now check your email account on your mobile phone to download your new app.

                    cat lying on shelf
                    Article
                    Causes of Cat Allergies
                    Allergy prick test
                    VIDEO
                    Types of Allergy Tests
                     
                    Man sneezing into tissue
                    Assessment
                    Get Personalized Allergy & Sinus Tips
                    Woman holding feather duster up to face, twitching
                    Quiz
                    Test Your Indoor Allergy Smarts
                     
                    ================================================ FILE: test/test-pages/webmd-2/expected-metadata.json ================================================ { "title": "Superbugs: What They Are and How You Get Them", "byline": "By Kelli Miller\n WebMD Health News", "dir": null, "lang": "en", "excerpt": "Drug-resistant bacteria, dubbed", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/webmd-2/expected.html ================================================

                    April 17, 2015 -- Imagine being sick in the hospital with a bacterial infection and doctors can't stop it from spreading. This so-called "superbug" scenario is not science fiction. It's an urgent, worldwide worry that is prompting swift action.

                    Every year, about 2 million people get sick from a superbug, according to the CDC. About 23,000 die. Earlier this year, an outbreak of CRE (carbapenem-resistant enterobacteriaceae) linked to contaminated medical tools sickened 11 people at two Los-Angeles area hospitals. Two people died, and more than 200 others may have been exposed.

                    The White House recently released a comprehensive plan outlining steps to combat drug-resistant bacteria. The plan identifies three "urgent" and several "serious" threats. We asked infectious disease experts to explain what some of them are and when to worry.

                    But First: What's a Superbug?

                    It's a term coined by the media to describe bacteria that cannot be killed using multiple antibiotics. "It resonates because it's scary," says Stephen Calderwood, MD, president of the Infectious Diseases Society of America. "But in fairness, there is no real definition."

                    Instead, doctors often use phrases like "multidrug-resistant bacteria." That's because a superbug isn't necessarily resistant to all antibiotics. It refers to bacteria that can't be treated using two or more, says Brian K. Coombes, PhD, of McMaster University in Ontario.

                    Any species of bacteria can turn into a superbug.

                    Misusing antibiotics (such as taking them when you don't need them or not finishing all of your medicine) is the "single leading factor" contributing to this problem, the CDC says. The concern is that eventually doctors will run out of antibiotics to treat them.

                    "What the public should know is that the more antibiotics you’ve taken, the higher your superbug risk," says Eric Biondi, MD, who runs a program to decrease unnecessary antibiotic use. "The more encounters you have with the hospital setting, the higher your superbug risk."

                    "Superbugs should be a concern to everyone," Coombes says. "Antibiotics are the foundation on which all modern medicine rests. Cancer chemotherapy, organ transplants, surgeries, and childbirth all rely on antibiotics to prevent infections. If you can't treat those, then we lose the medical advances we have made in the last 50 years."

                    Here are some of the growing superbug threats identified in the 2015 White House report.

                    ================================================ FILE: test/test-pages/webmd-2/source.html ================================================ Superbugs: What They Are and How You Get Them
                    Skip to content

                    Information and Resources

                    Font Size
                    A
                    A
                    A

                    Superbugs: What They Are and How You Get Them

                    By
                    WebMD Health News
                    Reviewed by Hansa D. Bhargava, MD

                    April 17, 2015 -- Imagine being sick in the hospital with a bacterial infection and doctors can't stop it from spreading. This so-called "superbug" scenario is not science fiction. It's an urgent, worldwide worry that is prompting swift action.

                    Every year, about 2 million people get sick from a superbug, according to the CDC. About 23,000 die. Earlier this year, an outbreak of CRE (carbapenem-resistant enterobacteriaceae) linked to contaminated medical tools sickened 11 people at two Los-Angeles area hospitals. Two people died, and more than 200 others may have been exposed.

                    The White House recently released a comprehensive plan outlining steps to combat drug-resistant bacteria. The plan identifies three "urgent" and several "serious" threats. We asked infectious disease experts to explain what some of them are and when to worry.

                    A Closer Look at MRSA

                    But First: What's a Superbug?

                    It's a term coined by the media to describe bacteria that cannot be killed using multiple antibiotics. "It resonates because it's scary," says Stephen Calderwood, MD, president of the Infectious Diseases Society of America. "But in fairness, there is no real definition."

                    Instead, doctors often use phrases like "multidrug-resistant bacteria." That's because a superbug isn't necessarily resistant to all antibiotics. It refers to bacteria that can't be treated using two or more, says Brian K. Coombes, PhD, of McMaster University in Ontario.

                    Any species of bacteria can turn into a superbug.

                    Misusing antibiotics (such as taking them when you don't need them or not finishing all of your medicine) is the "single leading factor" contributing to this problem, the CDC says. The concern is that eventually doctors will run out of antibiotics to treat them.

                    "What the public should know is that the more antibiotics you’ve taken, the higher your superbug risk," says Eric Biondi, MD, who runs a program to decrease unnecessary antibiotic use. "The more encounters you have with the hospital setting, the higher your superbug risk."

                    "Superbugs should be a concern to everyone," Coombes says. "Antibiotics are the foundation on which all modern medicine rests. Cancer chemotherapy, organ transplants, surgeries, and childbirth all rely on antibiotics to prevent infections. If you can't treat those, then we lose the medical advances we have made in the last 50 years."

                    Here are some of the growing superbug threats identified in the 2015 White House report.

                    1 | 2 | 3
                    Next Page >

                    WebMD Video: Now Playing

                    click to hide section Dirty Truth About Hand Washing

                    Click here to wach video: Dirty Truth About Hand Washing

                    Which sex is the worst about washing up? Why is it so important? We’ve got the dirty truth on how and when to wash your hands.

                    Click here to watch video: Dirty Truth About Hand Washing

                    click to expand section Truth About Bad Breath
                    click to expand section Dangers of Gum Disease
                    click to expand section Truth About Shampoo and Conditioner
                    click to expand section Laugh Therapy: Does it Work?
                    ================================================ FILE: test/test-pages/wikia/expected-metadata.json ================================================ { "title": "'Star Wars' Original Cuts Might Get Released for 40th Anniversary", "byline": "James Akinaka", "dir": null, "lang": "en", "excerpt": "As a 40th birthday present to the Star Wars Saga and its fans, Lucasfilm could re-release the original versions of the original trilogy films.", "siteName": "Fandom powered by Wikia", "publishedTime": "2017-02-23T17:18:13-08:00", "readerable": true } ================================================ FILE: test/test-pages/wikia/expected.html ================================================

                    Although Lucasfilm is already planning a birthday bash for the Star Wars Saga at Celebration Orlando this April, fans might get another present for the saga’s 40th anniversary. According to fan site MakingStarWars.net, rumors abound that Lucasfilm might re-release the unaltered cuts of the saga’s original trilogy.

                    If the rumors are true, this is big news for Star Wars fans. Aside from limited VHS releases, the unaltered cuts of the original trilogy films haven’t been available since they premiered in theaters in the 1970s and ’80s. If Lucasfilm indeed re-releases the films’ original cuts, then this will be the first time in decades that fans can see the films in their original forms. Here’s what makes the unaltered cuts of the original trilogy so special.

                    The Star Wars Special Editions Caused Controversy star wars han solo

                    Thanks to the commercial success of Star Wars, George Lucas has revisited and further edited his films for re-releases. The most notable — and controversial — release were the Special Editions of the original trilogy. In 1997, to celebrate the saga’s 20th anniversary, Lucasfilm spent a total of $15 million to remaster A New Hope, The Empire Strikes Back, and Return of the Jedi. The Special Editions had stints in theaters before moving to home media.

                    Although most of the Special Editions’ changes were cosmetic, others significantly affected the plot of the films. The most notable example is the “Han shot first” scene in A New Hope. As a result, the Special Editions generated significant controversy among Star Wars fans. Many fans remain skeptical about George Lucas’s decision to finish each original trilogy film “the way it was meant to be.”

                    star wars

                    While the Special Editions represent the most significant edits to the original trilogy, the saga has undergone other changes. Following up on the saga’s first Blu-ray release in 2011, Industrial Light & Magic (ILM) began remastering the entire saga in 3D, starting with the prequel trilogy. The Phantom Menace saw a theatrical 3D re-release in 2012, but Disney’s 2012 acquisition of Lucasfilm indefinitely postponed further 3D releases.

                    In 2015, Attack of the Clones and Revenge of the Sith received limited 3D showings at Celebration Anaheim. Other than that, it seems as though Disney has decided to refocus Lucasfilm’s efforts to new films. Of course, that’s why the saga has produced new content beginning with The Force Awakens. However, it looks like Lucasfilm isn’t likely to generate 3D versions of the original trilogy anytime soon.

                    Why the Original Film Cuts Matter

                    Admittedly, the differences between the original trilogy’s unaltered cuts and the Special Editions appeal to more hardcore fans. Casual fans are less likely to care about whether Greedo or Han Solo shot first. Still, given Star Wars’ indelible impact on pop culture, there’s certainly a market for the original trilogy’s unaltered cuts. They might not be for every Star Wars fan, but many of us care about them.

                    ILM supervisor John Knoll, who first pitched the story idea for Rogue One, said last year that ILM finished a brand new 4K restoration print of A New Hope. For that reason, it seems likely that Lucasfilm will finally give diehard fans the original film cuts that they’ve clamored for. There’s no word yet whether the unaltered cuts will be released in theaters or on home media. At the very least, however, fans will likely get them after all this time. After all, the Special Editions marked the saga’s 20th anniversary. Star Wars turns 40 years old this year, so there’s no telling what’s in store.

                    Would you like to be part of the Fandom team? Join our Fan Contributor Program and share your voice on Fandom.com!

                    ================================================ FILE: test/test-pages/wikia/source.html ================================================ 'Star Wars' Original Cuts Might Get Released for 40th Anniversary | Fandom powered by Wikia

                    Introducing Fandom Author Profiles

                    Bringing you closer to your readers with a customizable profile.

                    Edit Your Profile
                    Games Movies TV
                    Wikis
                    Start a Wiki
                    Top Stories
                    Not Everyone Will Love 'For Honor', But That's a Good Thing
                    5 Quick-Thinking Icons in Pop Culture
                    'Overwatch's' Upcoming Character - Everything We Know
                    'Final Fantasy XIV' Offers Up New Look at Stormblood DLC
                    Top Stories
                    Everything We Know About the Han Solo Movie [UPDATED]
                    'Alien: Covenant' Shows Off Its Corpses... Err, Cast
                    'Rogue One' Digital/Blu-ray Details and Extras Released
                    'Logan' Review - A Solid Step Forward for Superhero Movies
                    Top Stories
                    How 'Arrow' Handled Gun Control and Why Oliver Queen Is the Mayor We Need
                    The 4 Essential Ingredients of a Full-Fledged Fan Theory
                    Marvel's 'Iron Fist': Watch Danny Rand Use the Fire of the Fist
                    First Look at M. Night Shyamalan's 'Tales From the Crypt' Reboot Is Dead Wrong
                    Sign In | Register
                    My Profile
                    Log Out

                    TV

                    Games

                    Movies

                    ‘Star Wars’ Original Cuts Might Get Released for 40th Anniversary

                    James Akinaka •

                    Although Lucasfilm is already planning a birthday bash for the Star Wars Saga at Celebration Orlando this April, fans might get another present for the saga’s 40th anniversary. According to fan site MakingStarWars.net, rumors abound that Lucasfilm might re-release the unaltered cuts of the saga’s original trilogy.

                    If the rumors are true, this is big news for Star Wars fans. Aside from limited VHS releases, the unaltered cuts of the original trilogy films haven’t been available since they premiered in theaters in the 1970s and ’80s. If Lucasfilm indeed re-releases the films’ original cuts, then this will be the first time in decades that fans can see the films in their original forms. Here’s what makes the unaltered cuts of the original trilogy so special.

                    The Star Wars Special Editions Caused Controversy star wars han solo

                    Thanks to the commercial success of Star Wars, George Lucas has revisited and further edited his films for re-releases. The most notable — and controversial — release were the Special Editions of the original trilogy. In 1997, to celebrate the saga’s 20th anniversary, Lucasfilm spent a total of $15 million to remaster A New Hope, The Empire Strikes Back, and Return of the Jedi. The Special Editions had stints in theaters before moving to home media.

                    Although most of the Special Editions’ changes were cosmetic, others significantly affected the plot of the films. The most notable example is the “Han shot first” scene in A New Hope. As a result, the Special Editions generated significant controversy among Star Wars fans. Many fans remain skeptical about George Lucas’s decision to finish each original trilogy film “the way it was meant to be.”

                    star wars

                    While the Special Editions represent the most significant edits to the original trilogy, the saga has undergone other changes. Following up on the saga’s first Blu-ray release in 2011, Industrial Light & Magic (ILM) began remastering the entire saga in 3D, starting with the prequel trilogy. The Phantom Menace saw a theatrical 3D re-release in 2012, but Disney’s 2012 acquisition of Lucasfilm indefinitely postponed further 3D releases.

                    In 2015, Attack of the Clones and Revenge of the Sith received limited 3D showings at Celebration Anaheim. Other than that, it seems as though Disney has decided to refocus Lucasfilm’s efforts to new films. Of course, that’s why the saga has produced new content beginning with The Force Awakens. However, it looks like Lucasfilm isn’t likely to generate 3D versions of the original trilogy anytime soon.

                    Why the Original Film Cuts Matter

                    Admittedly, the differences between the original trilogy’s unaltered cuts and the Special Editions appeal to more hardcore fans. Casual fans are less likely to care about whether Greedo or Han Solo shot first. Still, given Star Wars’ indelible impact on pop culture, there’s certainly a market for the original trilogy’s unaltered cuts. They might not be for every Star Wars fan, but many of us care about them.

                    ILM supervisor John Knoll, who first pitched the story idea for Rogue One, said last year that ILM finished a brand new 4K restoration print of A New Hope. For that reason, it seems likely that Lucasfilm will finally give diehard fans the original film cuts that they’ve clamored for. There’s no word yet whether the unaltered cuts will be released in theaters or on home media. At the very least, however, fans will likely get them after all this time. After all, the Special Editions marked the saga’s 20th anniversary. Star Wars turns 40 years old this year, so there’s no telling what’s in store.

                    Would you like to be part of the Fandom team? Join our Fan Contributor Program and share your voice on Fandom.com!

                    James Akinaka

                    James Akinaka arrives at Fandom by way of Wookieepedia. He covers Star Wars, superheroes, and animation and values bold, inclusive stories. He suffers from a lifelong case of nitpicking and high standards.

                    View Profile

                    Latest Videos

                    See Batman and Superman Battle in Newest ‘Injustice 2’ Trailer

                    See Batman and Superman Battle in Newest ‘Injustice 2’ Trailer

                    ‘Ghost Recon: Wildlands’ Open Beta is Now Live

                    ‘Ghost Recon: Wildlands’ Open Beta is Now Live

                    Watch the 5-Minute Prologue to ‘Alien: Covenant’

                    Watch the 5-Minute Prologue to ‘Alien: Covenant’

                    Free Xbox Games With Gold Revealed For March 2017

                    Free Xbox Games With Gold Revealed For March 2017

                    ‘Dragon Quest Heroes II’ Release Date Revealed

                    ‘Dragon Quest Heroes II’ Release Date Revealed

                    Fan Feed

                    Fandom's Spring 2017 Games Guide Sponsored

                    Fandom’s Spring 2017 Games Guide

                    Henry Gilbert •
                    What's in Store for 'Young Justice' Season 3? [Updated] TV

                    What’s in Store for ‘Young Justice’ Season 3? [Updated]

                    James Akinaka •
                    Who Are 'The Walking Dead' Junkyard Gang? TV

                    Who Are ‘The Walking Dead’ Junkyard Gang?

                    Brian Linder •
                    What Did We Learn from the Nintendo Switch Leak? Games

                    What Did We Learn from the Nintendo Switch Leak?

                    Bob Mackey •
                    'A Cure For Wellness' Review: A Wild, Crazy and Unpredictable Ride Movies

                    ‘A Cure For Wellness’ Review: A Wild, Crazy and Unpredictable Ride

                    Kim Taylor-Foster •
                    (UPDATE) It Looks Like Matt Reeves WILL Direct 'The Batman' Movies

                    (UPDATE) It Looks Like Matt Reeves WILL Direct ‘The Batman’

                    Brandon Marcus •
                    What Surprises Does Marvel Have in Store For Guardians of the Galaxy Vol. 2? Movies

                    What Surprises Does Marvel Have in Store For Guardians of the Galaxy Vol. 2?

                    Kim Taylor-Foster •
                    'The Great Wall' Review - An Exciting but Empty Spectacle Movies

                    ‘The Great Wall’ Review – An Exciting but Empty Spectacle

                    Drew Dietsch •
                    Not Everyone Will Love 'For Honor', But That's a Good Thing Games

                    Not Everyone Will Love ‘For Honor’, But That’s a Good Thing

                    Samantha Loveridge •
                    'Star Wars: The Last Jedi' - The Title Has More to Tell Us Movies

                    ‘Star Wars: The Last Jedi’ – The Title Has More to Tell Us

                    Drew Dietsch •
                    5 Quick-Thinking Icons in Pop Culture Games

                    5 Quick-Thinking Icons in Pop Culture

                    Kim Taylor-Foster •
                    Everything We Know About the Han Solo Movie [UPDATED] Movies

                    Everything We Know About the Han Solo Movie [UPDATED]

                    Chris Tilly •
                    Got it! Your favorite fandoms are coming to your inbox.
                    ================================================ FILE: test/test-pages/wikipedia/expected-metadata.json ================================================ { "title": "Mozilla - Wikipedia", "byline": null, "dir": "ltr", "lang": "en", "excerpt": "Mozilla is a free-software community, created in 1998 by members of Netscape. The Mozilla community uses, develops, spreads and supports Mozilla products, thereby promoting exclusively free software and open standards, with only minor exceptions.[1] The community is supported institutionally by the Mozilla Foundation and its tax-paying subsidiary, the Mozilla Corporation.[2]", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/wikipedia/expected.html ================================================
                    Mozilla
                    Mozilla dinosaur head logo.png
                    Industry Open-source software
                    Founded February 28, 1998; 18 years ago
                    Founder Netscape Communications Corporation
                    Products Mozilla Application Suite
                    Divisions
                    Website mozilla.org/,%20https://www.mozilla.org/tr/

                    Mozilla is a free-software community, created in 1998 by members of Netscape. The Mozilla community uses, develops, spreads and supports Mozilla products, thereby promoting exclusively free software and open standards, with only minor exceptions.[1] The community is supported institutionally by the Mozilla Foundation and its tax-paying subsidiary, the Mozilla Corporation.[2]

                    Mozilla produces many products such as the Firefox web browser, Thunderbird e-mail client, Firefox Mobile web browser, Firefox OS mobile operating system, Bugzilla bug tracking system and other projects.

                    History[edit]

                    On January 23, 1998, Netscape made two announcements: first, that Netscape Communicator will be free; second, that the source code will also be free.[3] One day later, Jamie Zawinski from Netscape registered mozilla.org.[4] The project was named Mozilla after the original code name of the Netscape Navigator browser which is a blending of "Mosaic and Godzilla"[5] and used to co-ordinate the development of the Mozilla Application Suite, the open source version of Netscape's internet software, Netscape Communicator.[6][7] Jamie Zawinski says he came up with the name "Mozilla" at a Netscape staff meeting.[8][9] A small group of Netscape employees were tasked with coordination of the new community.

                    Originally, Mozilla aimed to be a technology provider for companies, such as Netscape, who would commercialize their open source code.[10] When AOL (Netscape's parent company) greatly reduced its involvement with Mozilla in July 2003, the Mozilla Foundation was designated the legal steward of the project.[11] Soon after, Mozilla deprecated the Mozilla Suite in favor of creating independent applications for each function, primarily the Firefox web browser and the Thunderbird email client, and moved to supply them directly to the public.[12]

                    Recently, Mozilla's activities have expanded to include Firefox on mobile platforms (primarily Android),[13] a mobile OS called Firefox OS,[14] a web-based identity system called Mozilla Persona and a marketplace for HTML5 applications.[15]

                    In a report released in November 2012, Mozilla reported that their total revenue for 2011 was $163 million, which was up 33% from $123 million in 2010. Mozilla noted that roughly 85% of their revenue comes from their contract with Google.[16]

                    At the end of 2013, Mozilla announced a deal with Cisco Systems whereby Firefox would download and use a Cisco-provided binary build of an open source[17] codec to play the proprietary H.264 video format.[18][19] As part of the deal, Cisco would pay any patent licensing fees associated with the binaries that it distributes. Mozilla's CTO, Brendan Eich, acknowledged that this is "not a complete solution" and isn't "perfect".[20] An employee in Mozilla's video formats team, writing in an unofficial capacity, justified[21] it by the need to maintain their large user base, which would be necessary in future battles for truly free video formats.

                    In December 2013, Mozilla announced funding for the development of non-free games[22] through its Game Creator Challenge. However, even those games that may be released under a non-free software or open source license must be made with open web technologies and Javascript as per the work criteria outlined in the announcement.

                    Eich CEO promotion controversy[edit]

                    On March 24, 2014, Mozilla promoted Brendan Eich to the role of CEO. This led to boycotts and protests from the LGBT community and its supporters, as Eich previously donated US$1,000[23] in 2008 in support of California's Proposition 8, a California ballot proposition and state constitutional amendment in opposition to same-sex marriage.[24] Eich's donation first became public knowledge in 2012, while he was Mozilla’s chief technical officer, leading to angry responses on Twitter—including the use of the hashtag "#wontworkwithbigots".[25]

                    Protests also emerged in 2014 following the announcement of Eich's appointment as CEO of Mozilla. U.S. companies OkCupid and CREDO Mobile received media coverage for their objections, with the former asking its users to boycott the browser,[26] while Credo amassed 50,000 signatures for a petition that called for Eich's resignation

                    Due to the controversy, Eich voluntarily stepped down on April 3, 2014[27] and Mitchell Baker, executive chairwoman of Mozilla Corporation, posted a statement on the Mozilla blog: "We didn’t move fast enough to engage with people once the controversy started. Mozilla believes both in equality and freedom of speech. Equality is necessary for meaningful speech. And you need free speech to fight for equality."[28] Eich's resignation promoted a larger backlash from conservatives who felt he had been forced out of the company internally.[citation needed]

                    OkCupid co-founder and CEO Sam Yagan had also donated $500[29] to Republican candidate Chris Cannon who proceeded to vote for multiple measures viewed as "anti-gay", including the banning of same-sex marriage.[30][31][32][33] Yagan claims he did not know about Cannon's stance on gay rights and that his contribution was due to the candidate being the ranking Republican participating in the House subcommittee that oversaw Internet and Intellectual Property matters.[34][35][36][37][38]

                    Reader comments on articles that were published close to the events were divided between support for OkCupid's actions and opposition to them. Supporters claimed the boycott was justified and saw OkCupid's actions as a firm statement of opposition to intolerance towards the gay community. Opponents saw OkCupid's actions as hypocritical, since Eich is also the inventor of JavaScript, which is still required to browse OkCupid's website, and felt that users should not be punished for the actions of Mozilla and suspected that OkCupid's actions were a publicity stunt.[36][39]

                    Values[edit]

                    According to Mozilla's manifesto,[40] which outlines goals, principles, and a pledge, "The Mozilla project uses a community-based approach to create world-class open source software and to develop new types of collaborative activities". Mozilla's manifesto mentions only its beliefs in regards to the Internet and Internet privacy, and has no mention of any political or social viewpoints.

                    Pledge[edit]

                    According to the Mozilla Foundation:[41]

                    The Mozilla Foundation pledges to support the Mozilla Manifesto in its activities. Specifically, we will:

                    • Build and enable open-source technologies and communities that support the Manifesto’s principles;
                    • Build and deliver great consumer products that support the Manifesto’s principles;
                    • Use the Mozilla assets (intellectual property such as copyrights and trademarks, infrastructure, funds, and reputation) to keep the Internet an open platform;
                    • Promote models for creating economic value for the public benefit; and
                    • Promote the Mozilla Manifesto principles in public discourse and within the Internet industry.

                    Software[edit]

                    Firefox logo

                    Firefox[edit]

                    Firefox is a web browser, and is Mozilla's flagship software product. It is available in both desktop and mobile versions. Firefox uses the Gecko layout engine to render web pages, which implements current and anticipated web standards.[42] As of late 2015, Firefox has approximately 10-11% of worldwide usage share of web browsers, making it the 4th most-used web browser.[43][44][45]

                    Firefox began as an experimental branch of the Mozilla codebase by Dave Hyatt, Joe Hewitt and Blake Ross. They believed the commercial requirements of Netscape's sponsorship and developer-driven feature creep compromised the utility of the Mozilla browser.[46] To combat what they saw as the Mozilla Suite's software bloat, they created a stand-alone browser, with which they intended to replace the Mozilla Suite.

                    Firefox was originally named Phoenix but the name was changed so as to avoid trademark conflicts with Phoenix Technologies. The initially-announced replacement, Firebird, provoked objections from the Firebird project community.[47][48] The current name, Firefox, was chosen on February 9, 2004.[49]

                    Firefox Mobile[edit]

                    Firefox Mobile (codenamed Fennec) is the build of the Mozilla Firefox web browser for devices such as smartphones and tablet computers.

                    Firefox Mobile uses the same Gecko layout engine as Mozilla Firefox. For example, version 1.0 used the same engine as Firefox 3.6, and the following release, 4.0, shared core code with Firefox 4.0. Its features include HTML5 support, Firefox Sync, add-ons support and tabbed browsing.[50]

                    Firefox Mobile is currently available for Android 2.2 and above devices with an ARMv7 or ARMv6 CPU.[51] The x86 architecture is not officially supported.[52] Tristan Nitot, president of Mozilla Europe, has said that it's unlikely that an iPhone or a BlackBerry version will be released, citing Apple's iTunes Store application approval policies (which forbid applications competing with Apple's own, and forbid engines which run downloaded code) and BlackBerry's limited operating system as the reasons.[53]

                    Firefox OS[edit]

                    Firefox OS (project name: Boot to Gecko also known as B2G) is an open source operating system in development by Mozilla that aims to support HTML5 apps written using "open Web" technologies rather than platform-specific native APIs. The concept behind Firefox OS is that all user-accessible software will be HTML5 applications, that use Open Web APIs to access the phone's hardware directly via JavaScript.[54]

                    Some devices using this OS include[55] Alcatel One Touch Fire, ZTE Open, LG Fireweb.

                    Thunderbird[edit]

                    Thunderbird is a free, open source, cross-platform email and news client developed by the volunteers of the Mozilla Community.

                    On July 16, 2012, Mitchell Baker announced that Mozilla's leadership had come to the conclusion that on-going stability was the most important thing for Thunderbird and that innovation in Thunderbird was no longer a priority for Mozilla. In that update Baker also suggested that Mozilla had provided a pathway for community to innovate around Thunderbird if the community chooses.[56]

                    SeaMonkey[edit]

                    SeaMonkey logo

                    SeaMonkey (formerly the Mozilla Application Suite) is a free and open source cross platform suite of Internet software components including a web browser component, a client for sending and receiving email and USENET newsgroup messages, an HTML editor (Mozilla Composer) and the ChatZilla IRC client.

                    On March 10, 2005, the Mozilla Foundation announced that it would not release any official versions of Mozilla Application Suite beyond 1.7.x, since it had now focused on the standalone applications Firefox and Thunderbird.[57] SeaMonkey is now maintained by the SeaMonkey Council, which has trademarked the SeaMonkey name with help from the Mozilla Foundation.[58] The Mozilla Foundation provides project hosting for the SeaMonkey developers.

                    Bugzilla[edit]

                    Bugzilla logo

                    Bugzilla is a web-based general-purpose bug tracking system, which was released as open source software by Netscape Communications in 1998 along with the rest of the Mozilla codebase, and is currently stewarded by Mozilla. It has been adopted by a variety of organizations for use as a bug tracking system for both free and open source software and proprietary projects and products, including the Mozilla Foundation, the Linux kernel, GNOME, KDE, Red Hat, Novell, Eclipse and LibreOffice.[59]

                    Components[edit]

                    NSS[edit]

                    Network Security Services (NSS) comprises a set of libraries designed to support cross-platform development of security-enabled client and server applications. NSS provides a complete open-source implementation of crypto libraries supporting SSL and S/MIME. NSS was previously tri-licensed under the Mozilla Public License 1.1, the GNU General Public License, and the GNU Lesser General Public License, but upgraded to GPL-compatible MPL 2.0.

                    AOL, Red Hat, Sun Microsystems/Oracle Corporation, Google and other companies and individual contributors have co-developed NSS and it is used in a wide range of non-Mozilla products including Evolution, Pidgin, and Apache OpenOffice.

                    SpiderMonkey[edit]

                    SpiderMonkey is the original JavaScript engine developed by Brendan Eich when he invented JavaScript in 1995 as a developer at Netscape. It became part of the Mozilla product family when Mozilla inherited Netscape's code-base in 1998. In 2011, Eich transferred the nominal ownership of the SpiderMonkey code and project to Dave Mandelin.[60]

                    SpiderMonkey is a cross-platform engine written in C++ which implements ECMAScript, a standard developed from JavaScript.[60][61] It comprises an interpreter, several just-in-time compilers, a decompiler and a garbage collector. Products which embed SpiderMonkey include Firefox, Thunderbird, SeaMonkey, and many non-Mozilla applications.[62]

                    Rhino[edit]

                    Rhino is an open source JavaScript engine managed by the Mozilla Foundation. It is developed entirely in Java. Rhino converts JavaScript scripts into Java classes. Rhino works in both compiled and interpreted mode.[63]

                    Gecko[edit]

                    Gecko is a layout engine that supports web pages written using HTML, SVG, and MathML. Gecko is written in C++ and uses NSPR for platform independence. Its source code is licensed under the Mozilla Public License.

                    Firefox uses Gecko both for rendering web pages and for rendering its user interface. Gecko is also used by Thunderbird, SeaMonkey, and many non-Mozilla applications.

                    Rust[edit]

                    Rust is a compiled programming language being developed by Mozilla Research. It is designed for safety, concurrency, and performance. Rust is intended for creating large and complex software which needs to be both safe against exploits and fast.

                    Rust is being used in an experimental layout engine, Servo, which is developed by Mozilla and Samsung. Servo is not used in any consumer-oriented browsers yet. However, the Servo project developers plan for parts of the Servo source code to be merged into Gecko, and Firefox, incrementally.[64][65]

                    XULRunner[edit]

                    XULRunner is a software platform and technology experiment by Mozilla, that allows applications built with the same technologies used by Firefox extensions (XPCOM, Javascript, HTML, CSS, XUL) to be run natively as desktop applications, without requiring Firefox to be installed on the user's machine. XULRunner binaries are available for the Windows, GNU/Linux and OS X operating systems, allowing such applications to be effectively cross platform.

                    pdf.js[edit]

                    Pdf.js is a library developed by Mozilla that allows in-browser rendering of pdf documents using the HTML5 Canvas and Javascript. It is included by default in recent versions of Firefox, allowing the browser to render pdf documents without requiring an external plugin; and it is available separately as an extension named "PDF Viewer" for Firefox for Android, SeaMonkey, and the Firefox versions which don't include it built-in. It can also be included as part of a website's scripts, to allow pdf rendering for any browser that implements the required HTML5 features and can run Javascript.

                    Shumway[edit]

                    Shumway is an open source replacement for the Adobe Flash Player, developed by Mozilla since 2012, using open web technologies as a replacement for Flash technologies. It uses Javascript and HTML5 Canvas elements to render Flash and execute Actionscript. It is included by default in Firefox Nightly and can be installed as an extension for any recent version of Firefox. The current implementation is limited in its capabilities to render Flash content outside simple projects.

                    Other activities[edit]

                    Mozilla VR[edit]

                    Mozilla VR is a team focused on bringing Virtual reality tools, specifications, and standards to the open Web.[66] Mozilla VR maintains A-Frame (VR), a web framework for building VR experiences, and works on advancing WebVR support within web browsers.

                    Mozilla Persona[edit]

                    Mozilla Persona is a secure, cross-browser website authentication mechanism which allows a user to use a single username and password (or other authentication method) to log in to multiple sites.[67] Mozilla Persona will be shutting down on November 30, 2016.[68]

                    Mozilla Location Service[edit]

                    This open source crowdsourced geolocation service was started by Mozilla in 2013 and offers a free API.

                    Webmaker[edit]

                    Mozilla Webmaker is Mozilla's educational initiative, Webmaker's goal is to "help millions of people move from using the web to making the web." As part of Mozilla’s non-profit mission, Webmaker aims "to help the world increase their understanding of the web, take greater control of their online lives, and create a more web literate planet."[69][70][70]

                    Mozilla Developer Network[edit]

                    Mozilla maintains a comprehensive developer documentation website called the Mozilla Developer Network which contains information about web technologies including HTML, CSS, SVG, JavaScript, as well Mozilla-specific information. In addition, Mozilla publishes a large number of videos about web technologies and the development of Mozilla projects on the Air Mozilla website.[71][72]

                    [edit]

                    The Mozilla Community consists of over 40,000 active contributors from across the globe[citation needed]. It includes both paid employees and volunteers who work towards the goals set forth[40] in the Mozilla Manifesto. Many of the sub-communities in Mozilla have formed around localization efforts for Mozilla Firefox, and the Mozilla web properties.

                    Local communities[edit]

                    Mozilla spaces, London

                    There are a number of sub-communities that exist based on their geographical locations, where contributors near each other work together on particular activities, such as localization, marketing, PR and user support.

                    Mozilla Reps[edit]

                    Mozilla Reps logo

                    The Mozilla Reps program aims to empower and support volunteer Mozillians who want to become official representatives of Mozilla in their region/locale.

                    The program provides a simple framework and a specific set of tools to help Mozillians to organize and/or attend events, recruit and mentor new contributors, document and share activities, and support their local communities better.

                    When joining the program, a Mozilla Rep agrees to take on the following responsibilities:

                    • Represent Mozilla in their country/region
                    • Promote the Mozilla Project and its mission
                    • Build on and support existing/future local community efforts and programs
                    • Inspire, recruit and support new contributors
                    • Support and mentor future Mozilla Reps
                    • Document clearly all their activities

                    Conferences and events[edit]

                    Mozilla Festival[edit]

                    Speakers from the Knight Foundation discuss the future of news at the 2011 Mozilla Festival in London.

                    The Mozilla Festival is an annual event where hundreds of passionate people explore the Web, learn together and make things that can change the world. With the emphasis on making—the mantra of the Festival is "less yack, more hack." Journalists, coders, filmmakers, designers, educators, gamers, makers, youth and anyone else, from all over the world, are encouraged to attend, with attendees from more than 40 countries, working together at the intersection between freedom, the Web, and that years theme.

                    The event revolves around design challenges which address key issues based on the chosen theme for that years festival. In previous years the Mozilla Festival has focused on Learning, and Media, with the 2012 festival being based around making. The titles of the festival revolve around the main theme, freedom (as in freedom of speech not free beer), and the Web.

                    MozCamps[edit]

                    MozCamps are the critical part of the Grow Mozilla initiative which aims to grow the Mozilla Community. These camps aim to bring core contributors from around the world together. They are intensive multi-day summits that include keynote speeches by Mozilla leadership, workshops and breakout sessions (led by paid and unpaid staff), and fun social outings. All of these activities combine to reward contributors for their hard work, engage them with new products and initiatives, and align all attendees on Mozilla's mission.

                    Mozilla Summit[edit]

                    Mozilla Summit are the global event with active contributors and Mozilla employees to develop a shared understanding of Mozilla's mission together. Over 2,000 people representing 90 countries and 114 languages gathered in Santa Clara, Toronto and Brussels in 2013. Mozilla has since its last summit in 2013 replaced summits with all-hands where both employees and volunteers come together to collaborate the event is a scaled down version of Mozilla Summit.

                    See also[edit]

                    References[edit]

                    1. ^ For exceptions, see "Values" section below
                    2. ^ "About the Mozilla Corporation". Mozilla Foundation.
                    3. ^ "Freeing the Source: The Story of Mozilla". Open Sources: Voices from the Open Source Revolution. Retrieved 2016-05-01.
                    4. ^ "Mozilla.org WHOIS, DNS, & Domain Info". DomainTools. Retrieved 1 May 2016.
                    5. ^ Payment, S. (2007). Marc Andreessen and Jim Clark: The Founders of Netscape. Rosen Publishing Group. ISBN 9781404207196.
                    6. ^ "Netscape Announces mozilla.org, a Dedicated Team and Web Site Supporting Development of Free Client Source Code". Netscape. Archived from the original on October 4, 2002. Retrieved 2012-08-21.
                    7. ^ "Mac vendors ponder Netscape gambit.". Macworld. 1 May 1998. Retrieved 2012-08-19.
                    8. ^ Zawinski, Jamie (1996). "nscp dorm". Retrieved 2007-10-12.
                    9. ^ Dave Titus with assistance from Andrew Wong. "How was Mozilla born".
                    10. ^ "Introduction to Mozilla Source Code". Mozilla. Retrieved 2012-08-18. However, mozilla.org wants to emphasize that these milestones are being produced for testing purposes only.
                    11. ^ "mozilla.org Announces Launch of the Mozilla Foundation to Lead Open-Source Browser Efforts". Retrieved 2012-08-18.
                    12. ^ Eich, Brendan; David Hyatt (April 2, 2003). "mozilla development roadmap". Mozilla. Retrieved 2009-08-02.
                    13. ^ "Better Browsing on Your Android Smartphone". AllThingsD. Retrieved 2012-08-18.
                    14. ^ "Mozilla Releases Test Version of Firefox OS". PC Magazine. Retrieved 2012-08-18.
                    15. ^ "Mozilla Marketplace is live, lets you run web apps like desktop programs". Engadget. Retrieved 2012-08-18.
                    16. ^ Lardinois, Frederic (November 15, 2012). "Mozilla Releases Annual Report For 2011: Revenue Up 33% To $163M, Majority From Google". techcrunch.com.
                    17. ^ "cisco/openh264 · GitHub". github.com. Retrieved 2014-04-05.
                    18. ^ "Mozilla will add H.264 to Firefox as Cisco makes eleventh-hour push for WebRTC's future — Tech News and Analysis". gigaom.com. Retrieved 2014-04-05.
                    19. ^ "Cisco to release open-source H.264 codec, Mozilla makes tactical retreat - TechRepublic". techrepublic.com. Retrieved 2014-04-05.
                    20. ^ "Video Interoperability on the Web Gets a Boost From Cisco's H.264 Codec". Of course, this is not a not a complete solution. In a perfect world, codecs, like other basic Internet technologies such as TCP/IP, HTTP, and HTML, would be fully open and free
                    21. ^ "Comments on Cisco, Mozilla, and H.264". By endorsing Cisco's plan, there's no getting around the fact that we've caved on our principles. That said, principles can't replace being in a practical position to make a difference in the future. - Christopher Montgomery wrote in a personal capacity but works for Mozilla in their codecs team
                    22. ^ "Game Creator Challenge -Contest Terms and Conditions". - submissions to the "amateur" category have to be released as free software, but not for the other two categories
                    23. ^ "Los Angeles Times - Brendan Eich contribution to Proposition 8". latimes.com. Retrieved 2014-07-01.
                    24. ^ "Gay Firefox developers boycott Mozilla to protest CEO hire [Updated] | Ars Technica". arstechnica.com. Retrieved 2014-04-05.
                    25. ^ Kelly Faircloth (9 April 2012). "Tech Celeb Makes Prop-8 Donation; Internet Goes Berserk". BetaBeat. BetaBeat. Retrieved 2014-04-28.
                    26. ^ "Screenshot of OkCupid's statement towards Firefox users". huffingtonpost.com. Retrieved 2014-07-01.
                    27. ^ "FAQ on CEO Resignation". The Mozilla Blog. Retrieved 2015-04-20.
                    28. ^ Baker, Mitchell (3 April 2014). "Brendan Eich Steps Down as Mozilla CEO". mozilla blog. Mozilla. Retrieved 2014-04-04.
                    29. ^ "opensecrets.org listing of Sam Yagan's contributions to political candidates". opensecrets.org. Retrieved 2014-07-01.
                    30. ^ "ontheissues.org listing of votes cast by Chris Cannon". ontheissues.org. Retrieved 2014-07-01.
                    31. ^ "ontheissues.org listing of votes cast on the permanency of the Patriot Act". ontheissues.org. Retrieved 2014-07-01.
                    32. ^ "ontheissues.org: Chris Cannon on Homeland Security". ontheissues.org. Retrieved 2014-07-01.
                    33. ^ "ontheissues.org: Chris Cannon on Abortion". ontheissues.org. Retrieved 2014-07-01.
                    34. ^ Levintova, Hannah (7 April 2014). "OkCupid's CEO Donated to an Anti-Gay Campaign Once, Too". Hanna Levintova article on motherjones.com. motherjones.com. Retrieved 2014-07-01.
                    35. ^ Lee, Stephanie M. (8 April 2014). "OKCupid CEO once donated to anti-gay politician". Stephanie M. Lee's blog on sfgate.com. sfgate.com. Retrieved 2014-07-01.
                    36. ^ a b "The Hypocrisy Of Sam Yagan & OkCupid". uncrunched.com blog. uncrunched.com. 6 April 2014. Retrieved 2014-07-01.
                    37. ^ Bellware, Kim (31 March 2014). "OKCupid Publicly Rips Mozilla: 'We Wish Them Nothing But Failure'". Kim Bellware article on huffingtonpost.com. huffingtonpost.com. Retrieved 2014-07-01.
                    38. ^ "Mozilla's Appointment Of Brendan Eich As CEO Sparks Controversy After Prop 8 Donation News Re-Emerges". huffingtonpost.com article. huffingtonpost.com. 27 March 2014. Retrieved 2014-07-01.
                    39. ^ Eidelson, Josh (4 April 2014). "OkCupid's gay rights stunt has its limits: Taking a deeper look at the savvy ploy". Josh Eidelson article on salon.com. salon.com. Retrieved 2014-07-01.
                    40. ^ a b "Mozilla Manifesto". Mozilla.org. Retrieved 2012-03-21.
                    41. ^ "The Mozilla Manifesto". Retrieved 24 July 2015.
                    42. ^ "Gecko Layout Engine". download-firefox.org. July 17, 2008. Archived from the original on 2010-11-28. Retrieved 2012-05-10.
                    43. ^ "Web Browser Market Share Trends". W3Counter. Awio Web Services LLC. Retrieved 2012-05-10.
                    44. ^ "Top 5 Browsers". StatCounter Global Stats. StatCounter. Retrieved 2012-05-10.
                    45. ^ "Web browsers (Global marketshare)". Clicky. Roxr Software Ltd. Retrieved 2012-05-10.
                    46. ^ Goodger, Ben (February 6, 2006). "Where Did Firefox Come From?". Inside Firefox. Archived from the original on 2011-06-23. Retrieved 2012-01-07.
                    47. ^ "Mozilla browser becomes Firebird". IBPhoenix. Archived from the original on 2007-09-14. Retrieved 2013-06-10. We at IBPhoenix think that having a browser and a database with the same name in the same space will confuse the market, especially as browsers and databases are often used in the same applications
                    48. ^ Festa, Paul (May 6, 2003). "Mozilla's Firebird gets wings clipped". CNET. Retrieved 2007-01-30.
                    49. ^ Festa, Paul (February 9, 2004). "Mozilla holds 'fire' in naming fight". CNET News. Retrieved 2007-01-24.
                    50. ^ "Mobile features". Mozilla. Retrieved 2012-06-26.
                    51. ^ "Mobile System Requirements".
                    52. ^ "Firefox Mobile supported devices".
                    53. ^ "Mozilla rules out Firefox for iPhone and BlackBerry".
                    54. ^ "Boot to Gecko Project". Mozilla. March 2012. Retrieved 2012-03-30.
                    55. ^ "Firefox OS - Devices & Availability". Mozilla. Retrieved 2015-12-30.
                    56. ^ "Thunderbird: Stability and Community Innovation | Mitchell's Blog". blog.lizardwrangler.com. Retrieved 2015-04-20.
                    57. ^ "Two discontinued browsers". LWN.net. 21 December 2005. Retrieved 2012-08-19.
                    58. ^ "SeaMonkey trademarks registered!". kairo.at. 2007-05-22. Retrieved 2013-06-10.
                    59. ^ "Bugzilla Installation List". Retrieved 2014-09-18.
                    60. ^ a b Eich, Brendan (21 June 2011). "New JavaScript Engine Module Owner". BrendanEich.com.
                    61. ^ "Bug 759422 - Remove use of e4x in account creation". Bugzilla@Mozilla. 2012-08-17. Retrieved 2012-08-18.
                    62. ^ "SpiderMonkey". Mozilla Developer Network. 2012-08-15. Retrieved 2012-08-18.
                    63. ^ "Rhino History". Mozilla Foundation. Retrieved 2008-03-20.
                    64. ^ "Roadmap". Retrieved 10 May 2016.
                    65. ^ Larabel, Michael. "Servo Continues Making Progress For Shipping Components In Gecko, Browser.html". Phoronix.com. Retrieved 10 May 2016.
                    66. ^ "Mozilla VR". Mozilla VR. Retrieved 2016-10-27.
                    67. ^ Persona, Mozilla
                    68. ^ "Persona". Mozilla Developer Network. Retrieved 2016-10-27.
                    69. ^ About Mozilla Webmaker, Mozilla
                    70. ^ a b Alan Henry. "Mozilla Webmaker Teaches You to Build Web Sites, Apps, and More". Lifehacker. Gawker Media.
                    71. ^ "Air Mozilla". Mozilla Wiki.
                    72. ^ "Air Mozilla Reboot, Phase I".

                    Constant downloads failure in firefox

                    External links[edit]

                    Wikimedia Commons has media related to Mozilla.
                    ================================================ FILE: test/test-pages/wikipedia/source.html ================================================ Mozilla - Wikipedia

                    Mozilla

                    From Wikipedia, the free encyclopedia
                    Jump to: navigation, search
                    See also: Mozilla Foundation and Mozilla Corporation
                    Mozilla
                    Industry Open-source software
                    Founded February 28, 1998; 18 years ago (1998-02-28)
                    Founder Netscape Communications Corporation
                    Products Mozilla Application Suite
                    Divisions
                    Website mozilla.org/,%20https://www.mozilla.org/tr/

                    Mozilla is a free-software community, created in 1998 by members of Netscape. The Mozilla community uses, develops, spreads and supports Mozilla products, thereby promoting exclusively free software and open standards, with only minor exceptions.[1] The community is supported institutionally by the Mozilla Foundation and its tax-paying subsidiary, the Mozilla Corporation.[2]

                    Mozilla produces many products such as the Firefox web browser, Thunderbird e-mail client, Firefox Mobile web browser, Firefox OS mobile operating system, Bugzilla bug tracking system and other projects.

                    History[edit]

                    On January 23, 1998, Netscape made two announcements: first, that Netscape Communicator will be free; second, that the source code will also be free.[3] One day later, Jamie Zawinski from Netscape registered mozilla.org.[4] The project was named Mozilla after the original code name of the Netscape Navigator browser which is a blending of "Mosaic and Godzilla"[5] and used to co-ordinate the development of the Mozilla Application Suite, the open source version of Netscape's internet software, Netscape Communicator.[6][7] Jamie Zawinski says he came up with the name "Mozilla" at a Netscape staff meeting.[8][9] A small group of Netscape employees were tasked with coordination of the new community.

                    Originally, Mozilla aimed to be a technology provider for companies, such as Netscape, who would commercialize their open source code.[10] When AOL (Netscape's parent company) greatly reduced its involvement with Mozilla in July 2003, the Mozilla Foundation was designated the legal steward of the project.[11] Soon after, Mozilla deprecated the Mozilla Suite in favor of creating independent applications for each function, primarily the Firefox web browser and the Thunderbird email client, and moved to supply them directly to the public.[12]

                    Recently, Mozilla's activities have expanded to include Firefox on mobile platforms (primarily Android),[13] a mobile OS called Firefox OS,[14] a web-based identity system called Mozilla Persona and a marketplace for HTML5 applications.[15]

                    In a report released in November 2012, Mozilla reported that their total revenue for 2011 was $163 million, which was up 33% from $123 million in 2010. Mozilla noted that roughly 85% of their revenue comes from their contract with Google.[16]

                    At the end of 2013, Mozilla announced a deal with Cisco Systems whereby Firefox would download and use a Cisco-provided binary build of an open source[17] codec to play the proprietary H.264 video format.[18][19] As part of the deal, Cisco would pay any patent licensing fees associated with the binaries that it distributes. Mozilla's CTO, Brendan Eich, acknowledged that this is "not a complete solution" and isn't "perfect".[20] An employee in Mozilla's video formats team, writing in an unofficial capacity, justified[21] it by the need to maintain their large user base, which would be necessary in future battles for truly free video formats.

                    In December 2013, Mozilla announced funding for the development of non-free games[22] through its Game Creator Challenge. However, even those games that may be released under a non-free software or open source license must be made with open web technologies and Javascript as per the work criteria outlined in the announcement.

                    Eich CEO promotion controversy[edit]

                    On March 24, 2014, Mozilla promoted Brendan Eich to the role of CEO. This led to boycotts and protests from the LGBT community and its supporters, as Eich previously donated US$1,000[23] in 2008 in support of California's Proposition 8, a California ballot proposition and state constitutional amendment in opposition to same-sex marriage.[24] Eich's donation first became public knowledge in 2012, while he was Mozilla’s chief technical officer, leading to angry responses on Twitter—including the use of the hashtag "#wontworkwithbigots".[25]

                    Protests also emerged in 2014 following the announcement of Eich's appointment as CEO of Mozilla. U.S. companies OkCupid and CREDO Mobile received media coverage for their objections, with the former asking its users to boycott the browser,[26] while Credo amassed 50,000 signatures for a petition that called for Eich's resignation

                    Due to the controversy, Eich voluntarily stepped down on April 3, 2014[27] and Mitchell Baker, executive chairwoman of Mozilla Corporation, posted a statement on the Mozilla blog: "We didn’t move fast enough to engage with people once the controversy started. Mozilla believes both in equality and freedom of speech. Equality is necessary for meaningful speech. And you need free speech to fight for equality."[28] Eich's resignation promoted a larger backlash from conservatives who felt he had been forced out of the company internally.[citation needed]

                    OkCupid co-founder and CEO Sam Yagan had also donated $500[29] to Republican candidate Chris Cannon who proceeded to vote for multiple measures viewed as "anti-gay", including the banning of same-sex marriage.[30][31][32][33] Yagan claims he did not know about Cannon's stance on gay rights and that his contribution was due to the candidate being the ranking Republican participating in the House subcommittee that oversaw Internet and Intellectual Property matters.[34][35][36][37][38]

                    Reader comments on articles that were published close to the events were divided between support for OkCupid's actions and opposition to them. Supporters claimed the boycott was justified and saw OkCupid's actions as a firm statement of opposition to intolerance towards the gay community. Opponents saw OkCupid's actions as hypocritical, since Eich is also the inventor of JavaScript, which is still required to browse OkCupid's website, and felt that users should not be punished for the actions of Mozilla and suspected that OkCupid's actions were a publicity stunt.[36][39]

                    Values[edit]

                    According to Mozilla's manifesto,[40] which outlines goals, principles, and a pledge, "The Mozilla project uses a community-based approach to create world-class open source software and to develop new types of collaborative activities". Mozilla's manifesto mentions only its beliefs in regards to the Internet and Internet privacy, and has no mention of any political or social viewpoints.

                    Pledge[edit]

                    According to the Mozilla Foundation:[41]

                    The Mozilla Foundation pledges to support the Mozilla Manifesto in its activities. Specifically, we will:

                    • Build and enable open-source technologies and communities that support the Manifesto’s principles;
                    • Build and deliver great consumer products that support the Manifesto’s principles;
                    • Use the Mozilla assets (intellectual property such as copyrights and trademarks, infrastructure, funds, and reputation) to keep the Internet an open platform;
                    • Promote models for creating economic value for the public benefit; and
                    • Promote the Mozilla Manifesto principles in public discourse and within the Internet industry.

                    Software[edit]

                    Firefox logo

                    Firefox[edit]

                    Main article: Firefox

                    Firefox is a web browser, and is Mozilla's flagship software product. It is available in both desktop and mobile versions. Firefox uses the Gecko layout engine to render web pages, which implements current and anticipated web standards.[42] As of late 2015, Firefox has approximately 10-11% of worldwide usage share of web browsers, making it the 4th most-used web browser.[43][44][45]

                    Firefox began as an experimental branch of the Mozilla codebase by Dave Hyatt, Joe Hewitt and Blake Ross. They believed the commercial requirements of Netscape's sponsorship and developer-driven feature creep compromised the utility of the Mozilla browser.[46] To combat what they saw as the Mozilla Suite's software bloat, they created a stand-alone browser, with which they intended to replace the Mozilla Suite.

                    Firefox was originally named Phoenix but the name was changed so as to avoid trademark conflicts with Phoenix Technologies. The initially-announced replacement, Firebird, provoked objections from the Firebird project community.[47][48] The current name, Firefox, was chosen on February 9, 2004.[49]

                    Firefox Mobile[edit]

                    Main article: Firefox Mobile

                    Firefox Mobile (codenamed Fennec) is the build of the Mozilla Firefox web browser for devices such as smartphones and tablet computers.

                    Firefox Mobile uses the same Gecko layout engine as Mozilla Firefox. For example, version 1.0 used the same engine as Firefox 3.6, and the following release, 4.0, shared core code with Firefox 4.0. Its features include HTML5 support, Firefox Sync, add-ons support and tabbed browsing.[50]

                    Firefox Mobile is currently available for Android 2.2 and above devices with an ARMv7 or ARMv6 CPU.[51] The x86 architecture is not officially supported.[52] Tristan Nitot, president of Mozilla Europe, has said that it's unlikely that an iPhone or a BlackBerry version will be released, citing Apple's iTunes Store application approval policies (which forbid applications competing with Apple's own, and forbid engines which run downloaded code) and BlackBerry's limited operating system as the reasons.[53]

                    Firefox OS[edit]

                    Main article: Firefox OS

                    Firefox OS (project name: Boot to Gecko also known as B2G) is an open source operating system in development by Mozilla that aims to support HTML5 apps written using "open Web" technologies rather than platform-specific native APIs. The concept behind Firefox OS is that all user-accessible software will be HTML5 applications, that use Open Web APIs to access the phone's hardware directly via JavaScript.[54]

                    Some devices using this OS include[55] Alcatel One Touch Fire, ZTE Open, LG Fireweb.

                    Thunderbird[edit]

                    Main article: Mozilla Thunderbird

                    Thunderbird is a free, open source, cross-platform email and news client developed by the volunteers of the Mozilla Community.

                    On July 16, 2012, Mitchell Baker announced that Mozilla's leadership had come to the conclusion that on-going stability was the most important thing for Thunderbird and that innovation in Thunderbird was no longer a priority for Mozilla. In that update Baker also suggested that Mozilla had provided a pathway for community to innovate around Thunderbird if the community chooses.[56]

                    SeaMonkey[edit]

                    Main article: SeaMonkey
                    SeaMonkey logo

                    SeaMonkey (formerly the Mozilla Application Suite) is a free and open source cross platform suite of Internet software components including a web browser component, a client for sending and receiving email and USENET newsgroup messages, an HTML editor (Mozilla Composer) and the ChatZilla IRC client.

                    On March 10, 2005, the Mozilla Foundation announced that it would not release any official versions of Mozilla Application Suite beyond 1.7.x, since it had now focused on the standalone applications Firefox and Thunderbird.[57] SeaMonkey is now maintained by the SeaMonkey Council, which has trademarked the SeaMonkey name with help from the Mozilla Foundation.[58] The Mozilla Foundation provides project hosting for the SeaMonkey developers.

                    Bugzilla[edit]

                    Main article: Bugzilla
                    Bugzilla logo

                    Bugzilla is a web-based general-purpose bug tracking system, which was released as open source software by Netscape Communications in 1998 along with the rest of the Mozilla codebase, and is currently stewarded by Mozilla. It has been adopted by a variety of organizations for use as a bug tracking system for both free and open source software and proprietary projects and products, including the Mozilla Foundation, the Linux kernel, GNOME, KDE, Red Hat, Novell, Eclipse and LibreOffice.[59]

                    Components[edit]

                    NSS[edit]

                    Main article: Network Security Services

                    Network Security Services (NSS) comprises a set of libraries designed to support cross-platform development of security-enabled client and server applications. NSS provides a complete open-source implementation of crypto libraries supporting SSL and S/MIME. NSS was previously tri-licensed under the Mozilla Public License 1.1, the GNU General Public License, and the GNU Lesser General Public License, but upgraded to GPL-compatible MPL 2.0.

                    AOL, Red Hat, Sun Microsystems/Oracle Corporation, Google and other companies and individual contributors have co-developed NSS and it is used in a wide range of non-Mozilla products including Evolution, Pidgin, and Apache OpenOffice.

                    SpiderMonkey[edit]

                    Main article: SpiderMonkey (JavaScript engine)

                    SpiderMonkey is the original JavaScript engine developed by Brendan Eich when he invented JavaScript in 1995 as a developer at Netscape. It became part of the Mozilla product family when Mozilla inherited Netscape's code-base in 1998. In 2011, Eich transferred the nominal ownership of the SpiderMonkey code and project to Dave Mandelin.[60]

                    SpiderMonkey is a cross-platform engine written in C++ which implements ECMAScript, a standard developed from JavaScript.[60][61] It comprises an interpreter, several just-in-time compilers, a decompiler and a garbage collector. Products which embed SpiderMonkey include Firefox, Thunderbird, SeaMonkey, and many non-Mozilla applications.[62]

                    Rhino[edit]

                    Main article: Rhino (JavaScript engine)

                    Rhino is an open source JavaScript engine managed by the Mozilla Foundation. It is developed entirely in Java. Rhino converts JavaScript scripts into Java classes. Rhino works in both compiled and interpreted mode.[63]

                    Gecko[edit]

                    Main article: Gecko (layout engine)

                    Gecko is a layout engine that supports web pages written using HTML, SVG, and MathML. Gecko is written in C++ and uses NSPR for platform independence. Its source code is licensed under the Mozilla Public License.

                    Firefox uses Gecko both for rendering web pages and for rendering its user interface. Gecko is also used by Thunderbird, SeaMonkey, and many non-Mozilla applications.

                    Rust[edit]

                    Main article: Rust (programming language)

                    Rust is a compiled programming language being developed by Mozilla Research. It is designed for safety, concurrency, and performance. Rust is intended for creating large and complex software which needs to be both safe against exploits and fast.

                    Rust is being used in an experimental layout engine, Servo, which is developed by Mozilla and Samsung. Servo is not used in any consumer-oriented browsers yet. However, the Servo project developers plan for parts of the Servo source code to be merged into Gecko, and Firefox, incrementally.[64][65]

                    XULRunner[edit]

                    Main article: XULRunner

                    XULRunner is a software platform and technology experiment by Mozilla, that allows applications built with the same technologies used by Firefox extensions (XPCOM, Javascript, HTML, CSS, XUL) to be run natively as desktop applications, without requiring Firefox to be installed on the user's machine. XULRunner binaries are available for the Windows, GNU/Linux and OS X operating systems, allowing such applications to be effectively cross platform.

                    pdf.js[edit]

                    Main article: pdf.js

                    Pdf.js is a library developed by Mozilla that allows in-browser rendering of pdf documents using the HTML5 Canvas and Javascript. It is included by default in recent versions of Firefox, allowing the browser to render pdf documents without requiring an external plugin; and it is available separately as an extension named "PDF Viewer" for Firefox for Android, SeaMonkey, and the Firefox versions which don't include it built-in. It can also be included as part of a website's scripts, to allow pdf rendering for any browser that implements the required HTML5 features and can run Javascript.

                    Shumway[edit]

                    Main article: Shumway (software)

                    Shumway is an open source replacement for the Adobe Flash Player, developed by Mozilla since 2012, using open web technologies as a replacement for Flash technologies. It uses Javascript and HTML5 Canvas elements to render Flash and execute Actionscript. It is included by default in Firefox Nightly and can be installed as an extension for any recent version of Firefox. The current implementation is limited in its capabilities to render Flash content outside simple projects.

                    Other activities[edit]

                    Mozilla VR[edit]

                    Mozilla VR is a team focused on bringing Virtual reality tools, specifications, and standards to the open Web.[66] Mozilla VR maintains A-Frame (VR), a web framework for building VR experiences, and works on advancing WebVR support within web browsers.

                    Mozilla Persona[edit]

                    Main article: Mozilla Persona

                    Mozilla Persona is a secure, cross-browser website authentication mechanism which allows a user to use a single username and password (or other authentication method) to log in to multiple sites.[67] Mozilla Persona will be shutting down on November 30, 2016.[68]

                    Mozilla Location Service[edit]

                    Main article: Mozilla Location Service

                    This open source crowdsourced geolocation service was started by Mozilla in 2013 and offers a free API.

                    Webmaker[edit]

                    Mozilla Webmaker is Mozilla's educational initiative, Webmaker's goal is to "help millions of people move from using the web to making the web." As part of Mozilla’s non-profit mission, Webmaker aims "to help the world increase their understanding of the web, take greater control of their online lives, and create a more web literate planet."[69][70][70]

                    Mozilla Developer Network[edit]

                    Main article: Mozilla Developer Network

                    Mozilla maintains a comprehensive developer documentation website called the Mozilla Developer Network which contains information about web technologies including HTML, CSS, SVG, JavaScript, as well Mozilla-specific information. In addition, Mozilla publishes a large number of videos about web technologies and the development of Mozilla projects on the Air Mozilla website.[71][72]

                    Community[edit]

                    The Mozilla Community consists of over 40,000 active contributors from across the globe[citation needed]. It includes both paid employees and volunteers who work towards the goals set forth[40] in the Mozilla Manifesto. Many of the sub-communities in Mozilla have formed around localization efforts for Mozilla Firefox, and the Mozilla web properties.

                    Local communities[edit]

                    Mozilla spaces, London

                    There are a number of sub-communities that exist based on their geographical locations, where contributors near each other work together on particular activities, such as localization, marketing, PR and user support.

                    Mozilla Reps[edit]

                    Mozilla Reps logo

                    The Mozilla Reps program aims to empower and support volunteer Mozillians who want to become official representatives of Mozilla in their region/locale.

                    The program provides a simple framework and a specific set of tools to help Mozillians to organize and/or attend events, recruit and mentor new contributors, document and share activities, and support their local communities better.

                    When joining the program, a Mozilla Rep agrees to take on the following responsibilities:

                    • Represent Mozilla in their country/region
                    • Promote the Mozilla Project and its mission
                    • Build on and support existing/future local community efforts and programs
                    • Inspire, recruit and support new contributors
                    • Support and mentor future Mozilla Reps
                    • Document clearly all their activities

                    Conferences and events[edit]

                    Mozilla Festival[edit]

                    Speakers from the Knight Foundation discuss the future of news at the 2011 Mozilla Festival in London.

                    The Mozilla Festival is an annual event where hundreds of passionate people explore the Web, learn together and make things that can change the world. With the emphasis on making—the mantra of the Festival is "less yack, more hack." Journalists, coders, filmmakers, designers, educators, gamers, makers, youth and anyone else, from all over the world, are encouraged to attend, with attendees from more than 40 countries, working together at the intersection between freedom, the Web, and that years theme.

                    The event revolves around design challenges which address key issues based on the chosen theme for that years festival. In previous years the Mozilla Festival has focused on Learning, and Media, with the 2012 festival being based around making. The titles of the festival revolve around the main theme, freedom (as in freedom of speech not free beer), and the Web.

                    MozCamps[edit]

                    MozCamps are the critical part of the Grow Mozilla initiative which aims to grow the Mozilla Community. These camps aim to bring core contributors from around the world together. They are intensive multi-day summits that include keynote speeches by Mozilla leadership, workshops and breakout sessions (led by paid and unpaid staff), and fun social outings. All of these activities combine to reward contributors for their hard work, engage them with new products and initiatives, and align all attendees on Mozilla's mission.

                    Mozilla Summit[edit]

                    Mozilla Summit are the global event with active contributors and Mozilla employees to develop a shared understanding of Mozilla's mission together. Over 2,000 people representing 90 countries and 114 languages gathered in Santa Clara, Toronto and Brussels in 2013. Mozilla has since its last summit in 2013 replaced summits with all-hands where both employees and volunteers come together to collaborate the event is a scaled down version of Mozilla Summit.

                    See also[edit]

                    References[edit]

                    1. ^ For exceptions, see "Values" section below
                    2. ^ "About the Mozilla Corporation". Mozilla Foundation. 
                    3. ^ "Freeing the Source: The Story of Mozilla". Open Sources: Voices from the Open Source Revolution. Retrieved 2016-05-01. 
                    4. ^ "Mozilla.org WHOIS, DNS, & Domain Info". DomainTools. Retrieved 1 May 2016. 
                    5. ^ Payment, S. (2007). Marc Andreessen and Jim Clark: The Founders of Netscape. Rosen Publishing Group. ISBN 9781404207196. 
                    6. ^ "Netscape Announces mozilla.org, a Dedicated Team and Web Site Supporting Development of Free Client Source Code". Netscape. Archived from the original on October 4, 2002. Retrieved 2012-08-21. 
                    7. ^ "Mac vendors ponder Netscape gambit.". Macworld. 1 May 1998. Retrieved 2012-08-19. 
                    8. ^ Zawinski, Jamie (1996). "nscp dorm". Retrieved 2007-10-12. 
                    9. ^ Dave Titus with assistance from Andrew Wong. "How was Mozilla born". 
                    10. ^ "Introduction to Mozilla Source Code". Mozilla. Retrieved 2012-08-18. However, mozilla.org wants to emphasize that these milestones are being produced for testing purposes only. 
                    11. ^ "mozilla.org Announces Launch of the Mozilla Foundation to Lead Open-Source Browser Efforts". Retrieved 2012-08-18. 
                    12. ^ Eich, Brendan; David Hyatt (April 2, 2003). "mozilla development roadmap". Mozilla. Retrieved 2009-08-02. 
                    13. ^ "Better Browsing on Your Android Smartphone". AllThingsD. Retrieved 2012-08-18. 
                    14. ^ "Mozilla Releases Test Version of Firefox OS". PC Magazine. Retrieved 2012-08-18. 
                    15. ^ "Mozilla Marketplace is live, lets you run web apps like desktop programs". Engadget. Retrieved 2012-08-18. 
                    16. ^ Lardinois, Frederic (November 15, 2012). "Mozilla Releases Annual Report For 2011: Revenue Up 33% To $163M, Majority From Google". techcrunch.com. 
                    17. ^ "cisco/openh264 · GitHub". github.com. Retrieved 2014-04-05. 
                    18. ^ "Mozilla will add H.264 to Firefox as Cisco makes eleventh-hour push for WebRTC's future — Tech News and Analysis". gigaom.com. Retrieved 2014-04-05. 
                    19. ^ "Cisco to release open-source H.264 codec, Mozilla makes tactical retreat - TechRepublic". techrepublic.com. Retrieved 2014-04-05. 
                    20. ^ "Video Interoperability on the Web Gets a Boost From Cisco's H.264 Codec". Of course, this is not a not a complete solution. In a perfect world, codecs, like other basic Internet technologies such as TCP/IP, HTTP, and HTML, would be fully open and free 
                    21. ^ "Comments on Cisco, Mozilla, and H.264". By endorsing Cisco's plan, there's no getting around the fact that we've caved on our principles. That said, principles can't replace being in a practical position to make a difference in the future.  - Christopher Montgomery wrote in a personal capacity but works for Mozilla in their codecs team
                    22. ^ "Game Creator Challenge -Contest Terms and Conditions".  - submissions to the "amateur" category have to be released as free software, but not for the other two categories
                    23. ^ "Los Angeles Times - Brendan Eich contribution to Proposition 8". latimes.com. Retrieved 2014-07-01. 
                    24. ^ "Gay Firefox developers boycott Mozilla to protest CEO hire [Updated] | Ars Technica". arstechnica.com. Retrieved 2014-04-05. 
                    25. ^ Kelly Faircloth (9 April 2012). "Tech Celeb Makes Prop-8 Donation; Internet Goes Berserk". BetaBeat. BetaBeat. Retrieved 2014-04-28. 
                    26. ^ "Screenshot of OkCupid's statement towards Firefox users". huffingtonpost.com. Retrieved 2014-07-01. 
                    27. ^ "FAQ on CEO Resignation". The Mozilla Blog. Retrieved 2015-04-20. 
                    28. ^ Baker, Mitchell (3 April 2014). "Brendan Eich Steps Down as Mozilla CEO". mozilla blog. Mozilla. Retrieved 2014-04-04. 
                    29. ^ "opensecrets.org listing of Sam Yagan's contributions to political candidates". opensecrets.org. Retrieved 2014-07-01. 
                    30. ^ "ontheissues.org listing of votes cast by Chris Cannon". ontheissues.org. Retrieved 2014-07-01. 
                    31. ^ "ontheissues.org listing of votes cast on the permanency of the Patriot Act". ontheissues.org. Retrieved 2014-07-01. 
                    32. ^ "ontheissues.org: Chris Cannon on Homeland Security". ontheissues.org. Retrieved 2014-07-01. 
                    33. ^ "ontheissues.org: Chris Cannon on Abortion". ontheissues.org. Retrieved 2014-07-01. 
                    34. ^ Levintova, Hannah (7 April 2014). "OkCupid's CEO Donated to an Anti-Gay Campaign Once, Too". Hanna Levintova article on motherjones.com. motherjones.com. Retrieved 2014-07-01. 
                    35. ^ Lee, Stephanie M. (8 April 2014). "OKCupid CEO once donated to anti-gay politician". Stephanie M. Lee's blog on sfgate.com. sfgate.com. Retrieved 2014-07-01. 
                    36. ^ a b "The Hypocrisy Of Sam Yagan & OkCupid". uncrunched.com blog. uncrunched.com. 6 April 2014. Retrieved 2014-07-01. 
                    37. ^ Bellware, Kim (31 March 2014). "OKCupid Publicly Rips Mozilla: 'We Wish Them Nothing But Failure'". Kim Bellware article on huffingtonpost.com. huffingtonpost.com. Retrieved 2014-07-01. 
                    38. ^ "Mozilla's Appointment Of Brendan Eich As CEO Sparks Controversy After Prop 8 Donation News Re-Emerges". huffingtonpost.com article. huffingtonpost.com. 27 March 2014. Retrieved 2014-07-01. 
                    39. ^ Eidelson, Josh (4 April 2014). "OkCupid's gay rights stunt has its limits: Taking a deeper look at the savvy ploy". Josh Eidelson article on salon.com. salon.com. Retrieved 2014-07-01. 
                    40. ^ a b "Mozilla Manifesto". Mozilla.org. Retrieved 2012-03-21. 
                    41. ^ "The Mozilla Manifesto". Retrieved 24 July 2015. 
                    42. ^ "Gecko Layout Engine". download-firefox.org. July 17, 2008. Archived from the original on 2010-11-28. Retrieved 2012-05-10. 
                    43. ^ "Web Browser Market Share Trends". W3Counter. Awio Web Services LLC. Retrieved 2012-05-10. 
                    44. ^ "Top 5 Browsers". StatCounter Global Stats. StatCounter. Retrieved 2012-05-10. 
                    45. ^ "Web browsers (Global marketshare)". Clicky. Roxr Software Ltd. Retrieved 2012-05-10. 
                    46. ^ Goodger, Ben (February 6, 2006). "Where Did Firefox Come From?". Inside Firefox. Archived from the original on 2011-06-23. Retrieved 2012-01-07. 
                    47. ^ "Mozilla browser becomes Firebird". IBPhoenix. Archived from the original on 2007-09-14. Retrieved 2013-06-10. We at IBPhoenix think that having a browser and a database with the same name in the same space will confuse the market, especially as browsers and databases are often used in the same applications 
                    48. ^ Festa, Paul (May 6, 2003). "Mozilla's Firebird gets wings clipped". CNET. Retrieved 2007-01-30. 
                    49. ^ Festa, Paul (February 9, 2004). "Mozilla holds 'fire' in naming fight". CNET News. Retrieved 2007-01-24. 
                    50. ^ "Mobile features". Mozilla. Retrieved 2012-06-26. 
                    51. ^ "Mobile System Requirements". 
                    52. ^ "Firefox Mobile supported devices". 
                    53. ^ "Mozilla rules out Firefox for iPhone and BlackBerry". 
                    54. ^ "Boot to Gecko Project". Mozilla. March 2012. Retrieved 2012-03-30. 
                    55. ^ "Firefox OS - Devices & Availability". Mozilla. Retrieved 2015-12-30. 
                    56. ^ "Thunderbird: Stability and Community Innovation | Mitchell's Blog". blog.lizardwrangler.com. Retrieved 2015-04-20. 
                    57. ^ "Two discontinued browsers". LWN.net. 21 December 2005. Retrieved 2012-08-19. 
                    58. ^ "SeaMonkey trademarks registered!". kairo.at. 2007-05-22. Retrieved 2013-06-10. 
                    59. ^ "Bugzilla Installation List". Retrieved 2014-09-18. 
                    60. ^ a b Eich, Brendan (21 June 2011). "New JavaScript Engine Module Owner". BrendanEich.com. 
                    61. ^ "Bug 759422 - Remove use of e4x in account creation". Bugzilla@Mozilla. 2012-08-17. Retrieved 2012-08-18. 
                    62. ^ "SpiderMonkey". Mozilla Developer Network. 2012-08-15. Retrieved 2012-08-18. 
                    63. ^ "Rhino History". Mozilla Foundation. Retrieved 2008-03-20. 
                    64. ^ "Roadmap". Retrieved 10 May 2016. 
                    65. ^ Larabel, Michael. "Servo Continues Making Progress For Shipping Components In Gecko, Browser.html". Phoronix.com. Retrieved 10 May 2016. 
                    66. ^ "Mozilla VR". Mozilla VR. Retrieved 2016-10-27. 
                    67. ^ Persona, Mozilla 
                    68. ^ "Persona". Mozilla Developer Network. Retrieved 2016-10-27. 
                    69. ^ About Mozilla Webmaker, Mozilla 
                    70. ^ a b Alan Henry. "Mozilla Webmaker Teaches You to Build Web Sites, Apps, and More". Lifehacker. Gawker Media. 
                    71. ^ "Air Mozilla". Mozilla Wiki. 
                    72. ^ "Air Mozilla Reboot, Phase I". 

                    Constant downloads failure in firefox

                    External links[edit]

                    Retrieved from "https://en.wikipedia.org/w/index.php?title=Mozilla&oldid=746574460"

                    Navigation menu

                    ================================================ FILE: test/test-pages/wikipedia-2/expected-metadata.json ================================================ { "title": "New Zealand", "byline": "Contributors to Wikimedia projects", "dir": "ltr", "lang": "en", "excerpt": "Coordinates: 42°S 174°E / 42°S 174°E", "siteName": "Wikimedia Foundation, Inc.", "publishedTime": "2001-10-29T01:59:14Z", "readerable": true } ================================================ FILE: test/test-pages/wikipedia-2/expected.html ================================================

                    Coordinates: 42°S 174°E / 42°S 174°E

                    New Zealand


                    Aotearoa  (Māori)

                    A map of the hemisphere centred on New Zealand, using an orthographic projection.

                    Location of New Zealand, including outlying islands, its territorial claim in the Antarctic, and Tokelau

                    Capital Wellington
                    41°17′S 174°27′E / 41.283°S 174.450°E
                    Largest city Auckland
                    Official languages
                    Ethnic groups

                    (2018)

                    Demonym(s) New Zealander
                    Kiwi (informal)
                    Government Unitary parliamentary constitutional monarchy

                    • Monarch

                    		Elizabeth II
                    Patsy Reddy
                    Jacinda Ardern
                    Legislature Parliament
                    (House of Representatives)
                    Stages of independence 

                    from the United Kingdom

                    7 May 1856

                    • Dominion

                    		26 September 1907

                    25 November 1947
                    Area

                    • Total

                    		268,021 km2 (103,483 sq mi) (75th)

                    • Water (%)

                    		1.6[n 4]
                    Population

                    • September 2019 estimate

                    		4,933,210[5] (120th)

                    • 2018 census

                    		4,699,755

                    • Density

                    		18.2/km2 (47.1/sq mi) (203rd)
                    GDP (PPP) 2018 estimate

                    • Total

                    		$199 billion[6]

                    • Per capita

                    		$40,266[6]
                    GDP (nominal) 2018 estimate

                    • Total

                    		$206 billion[6]

                    • Per capita

                    		$41,616[6]
                    Gini (2014) 33.0[7]
                    medium · 22nd
                    HDI (2017) Increase 0.917[8]
                    very high · 16th
                    Currency New Zealand dollar ($) (NZD)
                    Time zone UTC+12 (NZST[n 5])

                    • Summer (DST)

                    		UTC+13 (NZDT[n 6])
                    Date format dd/mm/yyyy
                    yyyy-mm-dd[10]
                    Driving side left
                    Calling code +64
                    ISO 3166 code NZ
                    Internet TLD .nz

                    New Zealand (Māori: Aotearoa [aɔˈtɛaɾɔa]) is a sovereign island country in the southwestern Pacific Ocean. The country geographically comprises two main landmasses—the North Island (Te Ika-a-Māui), and the South Island (Te Waipounamu)—and around 600 smaller islands. It has a total land area of 268,000 square kilometres (103,500 sq mi). New Zealand is situated some 2,000 kilometres (1,200 mi) east of Australia across the Tasman Sea and roughly 1,000 kilometres (600 mi) south of the Pacific island areas of New Caledonia, Fiji, and Tonga. Because of its remoteness, it was one of the last lands to be settled by humans. During its long period of isolation, New Zealand developed a distinct biodiversity of animal, fungal, and plant life. The country's varied topography and its sharp mountain peaks, such as the Southern Alps, owe much to the tectonic uplift of land and volcanic eruptions. New Zealand's capital city is Wellington, while its most populous city is Auckland.

                    Sometime between 1250 and 1300, Polynesians settled in the islands that later were named New Zealand and developed a distinctive Māori culture. In 1642, Dutch explorer Abel Tasman became the first European to sight New Zealand. In 1840, representatives of the United Kingdom and Māori chiefs signed the Treaty of Waitangi, which declared British sovereignty over the islands. In 1841, New Zealand became a colony within the British Empire and in 1907 it became a dominion; it gained full statutory independence in 1947 and the British monarch remained the head of state. Today, the majority of New Zealand's population of 4.9 million is of European descent; the indigenous Māori are the largest minority, followed by Asians and Pacific Islanders. Reflecting this, New Zealand's culture is mainly derived from Māori and early British settlers, with recent broadening arising from increased immigration. The official languages are English, Māori, and New Zealand Sign Language, with English being very dominant.

                    A developed country, New Zealand ranks highly in international comparisons of national performance, such as quality of life, health, education, protection of civil liberties, and economic freedom. New Zealand underwent major economic changes during the 1980s, which transformed it from a protectionist to a liberalised free-trade economy. The service sector dominates the national economy, followed by the industrial sector, and agriculture; international tourism is a significant source of revenue. Nationally, legislative authority is vested in an elected, unicameral Parliament, while executive political power is exercised by the Cabinet, led by the prime minister, currently Jacinda Ardern. Queen Elizabeth II is the country's monarch and is represented by a governor-general, currently Dame Patsy Reddy. In addition, New Zealand is organised into 11 regional councils and 67 territorial authorities for local government purposes. The Realm of New Zealand also includes Tokelau (a dependent territory); the Cook Islands and Niue (self-governing states in free association with New Zealand); and the Ross Dependency, which is New Zealand's territorial claim in Antarctica. New Zealand is a member of the United Nations, Commonwealth of Nations, ANZUS, Organisation for Economic Co-operation and Development, ASEAN Plus Six, Asia-Pacific Economic Cooperation, the Pacific Community and the Pacific Islands Forum.

                    Etymology

                    Brown square paper with Dutch writing and a thick red, curved line

                    Detail from a 1657 map showing the western coastline of "Nova Zeelandia". (In this map, north is at the bottom.)

                    Dutch explorer Abel Tasman sighted New Zealand in 1642 and named it Staten Land "in honour of the States General" (Dutch parliament). He wrote, "it is possible that this land joins to the Staten Land but it is uncertain",[11] referring to a landmass of the same name at the southern tip of South America, discovered by Jacob Le Maire in 1616.[12][13] In 1645, Dutch cartographers renamed the land Nova Zeelandia after the Dutch province of Zeeland.[14][15] British explorer James Cook subsequently anglicised the name to New Zealand.[16]

                    Aotearoa (pronounced ; often translated as "land of the long white cloud")[17] is the current Māori name for New Zealand. It is unknown whether Māori had a name for the whole country before the arrival of Europeans, with Aotearoa originally referring to just the North Island.[18] Māori had several traditional names for the two main islands, including Te Ika-a-Māui (the fish of Māui) for the North Island and Te Waipounamu (the waters of greenstone) or Te Waka o Aoraki (the canoe of Aoraki) for the South Island.[19] Early European maps labelled the islands North (North Island), Middle (South Island) and South (Stewart Island / Rakiura).[20] In 1830, mapmakers began to use "North" and "South" on their maps to distinguish the two largest islands and by 1907 this was the accepted norm.[16] The New Zealand Geographic Board discovered in 2009 that the names of the North Island and South Island had never been formalised, and names and alternative names were formalised in 2013. This set the names as North Island or Te Ika-a-Māui, and South Island or Te Waipounamu.[21] For each island, either its English or Māori name can be used, or both can be used together.[21]

                    History

                    One set of arrows point from Taiwan to Melanesia to Fiji/Samoa and then to the Marquesas Islands. The population then spread, some going south to New Zealand and others going north to Hawai'i. A second set start in southern Asia and end in Melanesia.

                    The Māori people are most likely descended from people who emigrated from Taiwan to Melanesia and then travelled east through to the Society Islands. After a pause of 70 to 265 years, a new wave of exploration led to the discovery and settlement of New Zealand.[22]

                    New Zealand was one of the last major landmasses settled by humans. Radiocarbon dating, evidence of deforestation[23] and mitochondrial DNA variability within Māori populations[24] suggest New Zealand was first settled by Eastern Polynesians between 1250 and 1300,[19][25] concluding a long series of voyages through the southern Pacific islands.[26] Over the centuries that followed, these settlers developed a distinct culture now known as Māori. The population was divided into iwi (tribes) and hapū (subtribes) who would sometimes cooperate, sometimes compete and sometimes fight against each other.[27] At some point a group of Māori migrated to Rēkohu, now known as the Chatham Islands, where they developed their distinct Moriori culture.[28][29] The Moriori population was all but wiped out between 1835 and 1862, largely because of Taranaki Māori invasion and enslavement in the 1830s, although European diseases also contributed. In 1862 only 101 survived, and the last known full-blooded Moriori died in 1933.[30]

                    An engraving of a sketched coastline on white background

                    Map of the New Zealand coastline as Cook charted it on his first visit in 1769–70. The track of the Endeavour is also shown.

                    The first Europeans known to have reached New Zealand were Dutch explorer Abel Tasman and his crew in 1642.[31] In a hostile encounter, four crew members were killed and at least one Māori was hit by canister shot.[32] Europeans did not revisit New Zealand until 1769 when British explorer James Cook mapped almost the entire coastline.[31] Following Cook, New Zealand was visited by numerous European and North American whaling, sealing and trading ships. They traded European food, metal tools, weapons and other goods for timber, Māori food, artefacts and water.[33] The introduction of the potato and the musket transformed Māori agriculture and warfare. Potatoes provided a reliable food surplus, which enabled longer and more sustained military campaigns.[34] The resulting intertribal Musket Wars encompassed over 600 battles between 1801 and 1840, killing 30,000–40,000 Māori.[35] From the early 19th century, Christian missionaries began to settle New Zealand, eventually converting most of the Māori population.[36] The Māori population declined to around 40% of its pre-contact level during the 19th century; introduced diseases were the major factor.[37]

                    A torn sheet of paper

                    In 1788 Captain Arthur Phillip assumed the position of Governor of the new British colony of New South Wales which according to his commission included New Zealand.[38] The British Government appointed James Busby as British Resident to New Zealand in 1832 following a petition from northern Māori.[39] In 1835, following an announcement of impending French settlement by Charles de Thierry, the nebulous United Tribes of New Zealand sent a Declaration of Independence to King William IV of the United Kingdom asking for protection.[39] Ongoing unrest, the proposed settlement of New Zealand by the New Zealand Company (which had already sent its first ship of surveyors to buy land from Māori) and the dubious legal standing of the Declaration of Independence prompted the Colonial Office to send Captain William Hobson to claim sovereignty for the United Kingdom and negotiate a treaty with the Māori.[40] The Treaty of Waitangi was first signed in the Bay of Islands on 6 February 1840.[41] In response to the New Zealand Company's attempts to establish an independent settlement in Wellington[42] and French settlers purchasing land in Akaroa,[43] Hobson declared British sovereignty over all of New Zealand on 21 May 1840, even though copies of the Treaty were still circulating throughout the country for Māori to sign.[44] With the signing of the Treaty and declaration of sovereignty the number of immigrants, particularly from the United Kingdom, began to increase.[45]

                    Black and white engraving depicting a crowd of people

                    New Zealand, still part of the colony of New South Wales, became a separate Colony of New Zealand on 1 July 1841.[46] Armed conflict began between the Colonial government and Māori in 1843 with the Wairau Affray over land and disagreements over sovereignty. These conflicts, mainly in the North Island, saw thousands of Imperial troops and the Royal Navy come to New Zealand and became known as the New Zealand Wars. Following these armed conflicts, large amounts of Māori land was confiscated by the government to meet settler demands.[47]

                    The colony gained a representative government in 1852 and the first Parliament met in 1854.[48] In 1856 the colony effectively became self-governing, gaining responsibility over all domestic matters other than native policy.[48] (Control over native policy was granted in the mid-1860s.[48]) Following concerns that the South Island might form a separate colony, premier Alfred Domett moved a resolution to transfer the capital from Auckland to a locality near Cook Strait.[49] Wellington was chosen for its central location, with Parliament officially sitting there for the first time in 1865.[50]

                    In 1891 the Liberal Party came to power as the first organised political party.[51] The Liberal Government, led by Richard Seddon for most of its period in office,[52] passed many important social and economic measures. In 1893 New Zealand was the first nation in the world to grant all women the right to vote[51] and in 1894 pioneered the adoption of compulsory arbitration between employers and unions.[53]

                    In 1907, at the request of the New Zealand Parliament, King Edward VII proclaimed New Zealand a Dominion within the British Empire,[54] reflecting its self-governing status.[55] In 1947 the country adopted the Statute of Westminster, confirming that the British Parliament could no longer legislate for New Zealand without the consent of New Zealand.[48]

                    Early in the 20th century, New Zealand was involved in world affairs, fighting in the First and Second World Wars[56] and suffering through the Great Depression.[57] The depression led to the election of the First Labour Government and the establishment of a comprehensive welfare state and a protectionist economy.[58] New Zealand experienced increasing prosperity following the Second World War[59] and Māori began to leave their traditional rural life and move to the cities in search of work.[60] A Māori protest movement developed, which criticised Eurocentrism and worked for greater recognition of Māori culture and of the Treaty of Waitangi.[61] In 1975, a Waitangi Tribunal was set up to investigate alleged breaches of the Treaty, and it was enabled to investigate historic grievances in 1985.[41] The government has negotiated settlements of these grievances with many iwi,[62] although Māori claims to the foreshore and seabed have proved controversial in the 2000s.[63][64]

                    Government and politics

                    The Queen wearing her New Zealand insignia

                    A smiling woman wearing a black dress

                    New Zealand is a constitutional monarchy with a parliamentary democracy,[65] although its constitution is not codified.[66] Elizabeth II is the Queen of New Zealand[67] and thus the head of state.[68] The Queen is represented by the governor-general, whom she appoints on the advice of the prime minister.[69] The governor-general can exercise the Crown's prerogative powers, such as reviewing cases of injustice and making appointments of ministers, ambassadors and other key public officials,[70] and in rare situations, the reserve powers (e.g. the power to dissolve parliament or refuse the royal assent of a bill into law).[71] The powers of the monarch and the governor-general are limited by constitutional constraints and they cannot normally be exercised without the advice of ministers.[71]

                    The New Zealand Parliament holds legislative power and consists of the Queen and the House of Representatives.[72] It also included an upper house, the Legislative Council, until this was abolished in 1950.[72] The supremacy of parliament over the Crown and other government institutions was established in England by the Bill of Rights 1689 and has been ratified as law in New Zealand.[72] The House of Representatives is democratically elected and a government is formed from the party or coalition with the majority of seats. If no majority is formed, a minority government can be formed if support from other parties during confidence and supply votes is assured.[72] The governor-general appoints ministers under advice from the prime minister, who is by convention the parliamentary leader of the governing party or coalition.[73] Cabinet, formed by ministers and led by the prime minister, is the highest policy-making body in government and responsible for deciding significant government actions.[74] Members of Cabinet make major decisions collectively, and are therefore collectively responsible for the consequences of these decisions.[75]

                    A parliamentary general election must be called no later than three years after the previous election.[76] Almost all general elections between 1853 and 1993 were held under the first-past-the-post voting system.[77] Since the 1996 election, a form of proportional representation called mixed-member proportional (MMP) has been used.[66] Under the MMP system, each person has two votes; one is for a candidate standing in the voter's electorate and the other is for a party. Since the 2014 election, there have been 71 electorates (which include seven Māori electorates in which only Māori can optionally vote),[78] and the remaining 49 of the 120 seats are assigned so that representation in parliament reflects the party vote, with the threshold that a party must win at least one electorate or 5% of the total party vote before it is eligible for a seat.[79]

                    A block of buildings fronted by a large statue.

                    Elections since the 1930s have been dominated by two political parties, National and Labour.[77] Between March 2005 and August 2006, New Zealand became the first country in the world in which all the highest offices in the land—head of state, governor-general, prime minister, speaker and chief justice—were occupied simultaneously by women.[80] The current prime minister is Jacinda Ardern, who has been in office since 26 October 2017.[81] She is the country's third female prime minister.[82]

                    New Zealand's judiciary, headed by the chief justice,[83] includes the Supreme Court, Court of Appeal, the High Court, and subordinate courts.[84] Judges and judicial officers are appointed non-politically and under strict rules regarding tenure to help maintain judicial independence.[66] This theoretically allows the judiciary to interpret the law based solely on the legislation enacted by Parliament without other influences on their decisions.[85]

                    New Zealand is identified as one of the world's most stable and well-governed states.[86] As at 2017, the country was ranked fourth in the strength of its democratic institutions,[87] and first in government transparency and lack of corruption.[88] A 2017 Human Rights Report by the U.S. Department of State noted that the government generally respected the rights of individuals, but voiced concerns regarding the social status of the Māori population.[89] New Zealand ranks highly for civic participation in the political process, with 77% voter turnout during recent elections, compared to an OECD average of 69%.[90]

                    Foreign relations and military

                    A squad of men kneel in the desert sand while performing a war dance

                    Early colonial New Zealand allowed the British Government to determine external trade and be responsible for foreign policy.[91] The 1923 and 1926 Imperial Conferences decided that New Zealand should be allowed to negotiate its own political treaties and the first commercial treaty was ratified in 1928 with Japan. On 3 September 1939 New Zealand allied itself with Britain and declared war on Germany with Prime Minister Michael Joseph Savage proclaiming, "Where she goes, we go; where she stands, we stand."[92]

                    In 1951 the United Kingdom became increasingly focused on its European interests,[93] while New Zealand joined Australia and the United States in the ANZUS security treaty.[94] The influence of the United States on New Zealand weakened following protests over the Vietnam War,[95] the refusal of the United States to admonish France after the sinking of the Rainbow Warrior,[96] disagreements over environmental and agricultural trade issues and New Zealand's nuclear-free policy.[97][98] Despite the United States' suspension of ANZUS obligations the treaty remained in effect between New Zealand and Australia, whose foreign policy has followed a similar historical trend.[99] Close political contact is maintained between the two countries, with free trade agreements and travel arrangements that allow citizens to visit, live and work in both countries without restrictions.[100] In 2013 there were about 650,000 New Zealand citizens living in Australia, which is equivalent to 15% of the resident population of New Zealand.[101]

                    A soldier in a green army uniform faces forwards

                    Anzac Day service at the National War Memorial

                    New Zealand has a strong presence among the Pacific Island countries. A large proportion of New Zealand's aid goes to these countries and many Pacific people migrate to New Zealand for employment.[102] Permanent migration is regulated under the 1970 Samoan Quota Scheme and the 2002 Pacific Access Category, which allow up to 1,100 Samoan nationals and up to 750 other Pacific Islanders respectively to become permanent New Zealand residents each year. A seasonal workers scheme for temporary migration was introduced in 2007 and in 2009 about 8,000 Pacific Islanders were employed under it.[103] A regional power,[104] New Zealand is involved in the Pacific Islands Forum, the Pacific Community, Asia-Pacific Economic Cooperation and the Association of Southeast Asian Nations Regional Forum (including the East Asia Summit).[100] New Zealand is a member of the United Nations,[105] the Commonwealth of Nations[106] and the Organisation for Economic Co-operation and Development (OECD),[107] and participates in the Five Power Defence Arrangements.[108]

                    New Zealand's military services—the Defence Force—comprise the New Zealand Army, the Royal New Zealand Air Force and the Royal New Zealand Navy.[109] New Zealand's national defence needs are modest, since a direct attack is unlikely.[110] However, its military has had a global presence. The country fought in both world wars, with notable campaigns in Gallipoli, Crete,[111] El Alamein[112] and Cassino.[113] The Gallipoli campaign played an important part in fostering New Zealand's national identity[114][115] and strengthened the ANZAC tradition it shares with Australia.[116]

                    In addition to Vietnam and the two world wars, New Zealand fought in the Second Boer War,[117] the Korean War,[118] the Malayan Emergency,[119] the Gulf War and the Afghanistan War. It has contributed forces to several regional and global peacekeeping missions, such as those in Cyprus, Somalia, Bosnia and Herzegovina, the Sinai, Angola, Cambodia, the Iran–Iraq border, Bougainville, East Timor, and the Solomon Islands.[120]

                    Local government and external territories

                    Map with the North, South, Stewart/Rakiura, Tokelau, Cook, Niue, Kermadec, Chatham, Bounty, Antipodes, Snare, Auckland and Campbell Islands highlighted. New Zealand's segment of Antarctica (the Ross Dependency) is also highlighted.

                    The early European settlers divided New Zealand into provinces, which had a degree of autonomy.[121] Because of financial pressures and the desire to consolidate railways, education, land sales and other policies, government was centralised and the provinces were abolished in 1876.[122] The provinces are remembered in regional public holidays[123] and sporting rivalries.[124]

                    Since 1876, various councils have administered local areas under legislation determined by the central government.[121][125] In 1989, the government reorganised local government into the current two-tier structure of regional councils and territorial authorities.[126] The 249 municipalities[126] that existed in 1975 have now been consolidated into 67 territorial authorities and 11 regional councils.[127] The regional councils' role is to regulate "the natural environment with particular emphasis on resource management",[126] while territorial authorities are responsible for sewage, water, local roads, building consents and other local matters.[128][129] Five of the territorial councils are unitary authorities and also act as regional councils.[129] The territorial authorities consist of 13 city councils, 53 district councils, and the Chatham Islands Council. While officially the Chatham Islands Council is not a unitary authority, it undertakes many functions of a regional council.[130]

                    The Realm of New Zealand, one of 16 Commonwealth realms,[131] is the entire area over which the Queen of New Zealand is sovereign, and comprises New Zealand, Tokelau, the Ross Dependency, the Cook Islands and Niue.[65] The Cook Islands and Niue are self-governing states in free association with New Zealand.[132][133] The New Zealand Parliament cannot pass legislation for these countries, but with their consent can act on behalf of them in foreign affairs and defence. Tokelau is classified as a non-self-governing territory, but is administered by a council of three elders (one from each Tokelauan atoll).[134] The Ross Dependency is New Zealand's territorial claim in Antarctica, where it operates the Scott Base research facility.[135] New Zealand nationality law treats all parts of the realm equally, so most people born in New Zealand, the Cook Islands, Niue, Tokelau and the Ross Dependency are New Zealand citizens.[136][n 7]

                    Administrative divisions of the Realm of New Zealand

                    Countries  New Zealand      Cook Islands  Niue
                    Regions 11 non-unitary regions 5 unitary regions Chatham Islands   Outlying islands outside any regional authority
                    (the Kermadec Islands, Three Kings Islands, and Subantarctic Islands)                     		Ross Dependency  Tokelau 15 islands 14 villages
                    Territorial authorities 13 cities and 53 districts
                    Notes Some districts lie in more than one region These combine the regional and the territorial authority levels in one Special territorial authority The outlying Solander Islands form part of the Southland Region New Zealand's Antarctic territory Non-self-governing territory of New Zealand States in free association with New Zealand

                    Environment

                    Geography

                    Islands of New Zealand as seen from satellite

                    New Zealand is located near the centre of the water hemisphere and is made up of two main islands and a number of smaller islands. The two main islands (the North Island, or Te Ika-a-Māui, and the South Island, or Te Waipounamu) are separated by Cook Strait, 22 kilometres (14 mi) wide at its narrowest point.[138] Besides the North and South Islands, the five largest inhabited islands are Stewart Island (across the Foveaux Strait), Chatham Island, Great Barrier Island (in the Hauraki Gulf),[139] D'Urville Island (in the Marlborough Sounds)[140] and Waiheke Island (about 22 km (14 mi) from central Auckland).[141]

                    A large mountain with a lake in the foreground

                    Snow-capped mountain range

                    The Southern Alps stretch for 500 kilometres down the South Island

                    New Zealand is long and narrow (over 1,600 kilometres (990 mi) along its north-north-east axis with a maximum width of 400 kilometres (250 mi)),[142] with about 15,000 km (9,300 mi) of coastline[143] and a total land area of 268,000 square kilometres (103,500 sq mi).[144] Because of its far-flung outlying islands and long coastline, the country has extensive marine resources. Its exclusive economic zone is one of the largest in the world, covering more than 15 times its land area.[145]

                    The South Island is the largest landmass of New Zealand. It is divided along its length by the Southern Alps.[146] There are 18 peaks over 3,000 metres (9,800 ft), the highest of which is Aoraki / Mount Cook at 3,754 metres (12,316 ft).[147] Fiordland's steep mountains and deep fiords record the extensive ice age glaciation of this southwestern corner of the South Island.[148] The North Island is less mountainous but is marked by volcanism.[149] The highly active Taupo Volcanic Zone has formed a large volcanic plateau, punctuated by the North Island's highest mountain, Mount Ruapehu (2,797 metres (9,177 ft)). The plateau also hosts the country's largest lake, Lake Taupo,[150] nestled in the caldera of one of the world's most active supervolcanoes.[151]

                    The country owes its varied topography, and perhaps even its emergence above the waves, to the dynamic boundary it straddles between the Pacific and Indo-Australian Plates.[152] New Zealand is part of Zealandia, a microcontinent nearly half the size of Australia that gradually submerged after breaking away from the Gondwanan supercontinent.[153] About 25 million years ago, a shift in plate tectonic movements began to contort and crumple the region. This is now most evident in the Southern Alps, formed by compression of the crust beside the Alpine Fault. Elsewhere the plate boundary involves the subduction of one plate under the other, producing the Puysegur Trench to the south, the Hikurangi Trench east of the North Island, and the Kermadec and Tonga Trenches[154] further north.[152]

                    New Zealand is part of a region known as Australasia, together with Australia.[155] It also forms the southwestern extremity of the geographic and ethnographic region called Polynesia.[156] The term Oceania is often used to denote the wider region encompassing the Australian continent, New Zealand and various islands in the Pacific Ocean that are not included in the seven-continent model.[157]

                    • Landscapes of New Zealand

                    Climate

                    New Zealand's climate is predominantly temperate maritime (Köppen: Cfb), with mean annual temperatures ranging from 10 °C (50 °F) in the south to 16 °C (61 °F) in the north.[158] Historical maxima and minima are 42.4 °C (108.32 °F) in Rangiora, Canterbury and −25.6 °C (−14.08 °F) in Ranfurly, Otago.[159] Conditions vary sharply across regions from extremely wet on the West Coast of the South Island to almost semi-arid in Central Otago and the Mackenzie Basin of inland Canterbury and subtropical in Northland.[160] Of the seven largest cities, Christchurch is the driest, receiving on average only 640 millimetres (25 in) of rain per year and Wellington the wettest, receiving almost twice that amount.[161] Auckland, Wellington and Christchurch all receive a yearly average of more than 2,000 hours of sunshine. The southern and southwestern parts of the South Island have a cooler and cloudier climate, with around 1,400–1,600 hours; the northern and northeastern parts of the South Island are the sunniest areas of the country and receive about 2,400–2,500 hours.[162] The general snow season is early June until early October, though cold snaps can occur outside this season.[163] Snowfall is common in the eastern and southern parts of the South Island and mountain areas across the country.[158]

                    The table below lists climate normals for the warmest and coldest months in New Zealand's six largest cities. North Island cities are generally warmest in February. South Island cities are warmest in January.

                    Average daily maximum and minimum temperatures for the six largest cities of New Zealand[164]
                    Location Jan/Feb (°C) Jan/Feb (°F) July (°C) July (°F)
                    Auckland 23/16 74/60 14/7 58/45
                    Wellington 20/13 68/56 11/6 52/42
                    Christchurch 22/12 72/53 10/0 51/33
                    Hamilton 24/13 75/56 14/4 57/39
                    Tauranga 24/15 75/59 14/6 58/42
                    Dunedin 19/11 66/53 10/3 50/37

                    Biodiversity

                    Kiwi amongst sticks

                    The endemic flightless kiwi is a national icon.

                    New Zealand's geographic isolation for 80 million years[165] and island biogeography has influenced evolution of the country's species of animals, fungi and plants. Physical isolation has caused biological isolation, resulting in a dynamic evolutionary ecology with examples of very distinctive plants and animals as well as populations of widespread species.[166][167] About 82% of New Zealand's indigenous vascular plants are endemic, covering 1,944 species across 65 genera.[168][169] The number of fungi recorded from New Zealand, including lichen-forming species, is not known, nor is the proportion of those fungi which are endemic, but one estimate suggests there are about 2,300 species of lichen-forming fungi in New Zealand[168] and 40% of these are endemic.[170] The two main types of forest are those dominated by broadleaf trees with emergent podocarps, or by southern beech in cooler climates.[171] The remaining vegetation types consist of grasslands, the majority of which are tussock.[172]

                    Before the arrival of humans, an estimated 80% of the land was covered in forest, with only high alpine, wet, infertile and volcanic areas without trees.[173] Massive deforestation occurred after humans arrived, with around half the forest cover lost to fire after Polynesian settlement.[174] Much of the remaining forest fell after European settlement, being logged or cleared to make room for pastoral farming, leaving forest occupying only 23% of the land.[175]

                    An artist's rendition of a Haast's eagle attacking two moa

                    The giant Haast's eagle died out when humans hunted its main prey, the moa, to extinction.

                    The forests were dominated by birds, and the lack of mammalian predators led to some like the kiwi, kakapo, weka and takahē evolving flightlessness.[176] The arrival of humans, associated changes to habitat, and the introduction of rats, ferrets and other mammals led to the extinction of many bird species, including large birds like the moa and Haast's eagle.[177][178]

                    Other indigenous animals are represented by reptiles (tuatara, skinks and geckos), frogs,[179] spiders,[180] insects (weta)[181] and snails.[182] Some, such as the tuatara, are so unique that they have been called living fossils.[183] Three species of bats (one since extinct) were the only sign of native land mammals in New Zealand until the 2006 discovery of bones from a unique, mouse-sized land mammal at least 16 million years old.[184][185] Marine mammals however are abundant, with almost half the world's cetaceans (whales, dolphins, and porpoises) and large numbers of fur seals reported in New Zealand waters.[186] Many seabirds breed in New Zealand, a third of them unique to the country.[187] More penguin species are found in New Zealand than in any other country.[188]

                    Since human arrival, almost half of the country's vertebrate species have become extinct, including at least fifty-one birds, three frogs, three lizards, one freshwater fish, and one bat. Others are endangered or have had their range severely reduced.[177] However, New Zealand conservationists have pioneered several methods to help threatened wildlife recover, including island sanctuaries, pest control, wildlife translocation, fostering, and ecological restoration of islands and other selected areas.[189][190][191][192]

                    Economy

                    Boats docked in blue-green water. Plate glass skyscrapers rising up in the background.

                    New Zealand has an advanced market economy,[193] ranked 16th in the 2018 Human Development Index[8] and third in the 2018 Index of Economic Freedom.[194] It is a high-income economy with a nominal gross domestic product (GDP) per capita of US$36,254.[6] The currency is the New Zealand dollar, informally known as the "Kiwi dollar"; it also circulates in the Cook Islands (see Cook Islands dollar), Niue, Tokelau, and the Pitcairn Islands.[195]

                    Historically, extractive industries have contributed strongly to New Zealand's economy, focussing at different times on sealing, whaling, flax, gold, kauri gum, and native timber.[196] The first shipment of refrigerated meat on the Dunedin in 1882 led to the establishment of meat and dairy exports to Britain, a trade which provided the basis for strong economic growth in New Zealand.[197] High demand for agricultural products from the United Kingdom and the United States helped New Zealanders achieve higher living standards than both Australia and Western Europe in the 1950s and 1960s.[198] In 1973, New Zealand's export market was reduced when the United Kingdom joined the European Economic Community[199] and other compounding factors, such as the 1973 oil and 1979 energy crises, led to a severe economic depression.[200] Living standards in New Zealand fell behind those of Australia and Western Europe, and by 1982 New Zealand had the lowest per-capita income of all the developed nations surveyed by the World Bank.[201] In the mid-1980s New Zealand deregulated its agricultural sector by phasing out subsidies over a three-year period.[202][203] Since 1984, successive governments engaged in major macroeconomic restructuring (known first as Rogernomics and then Ruthanasia), rapidly transforming New Zealand from a protected and highly regulated economy to a liberalised free-trade economy.[204][205]

                    Blue water against a backdrop of snow-capped mountains

                    Milford Sound is one of New Zealand's most famous tourist destinations.[206]

                    Unemployment peaked above 10% in 1991 and 1992,[207] following the 1987 share market crash, but eventually fell to a record low (since 1986) of 3.7% in 2007 (ranking third from twenty-seven comparable OECD nations).[207] However, the global financial crisis that followed had a major impact on New Zealand, with the GDP shrinking for five consecutive quarters, the longest recession in over thirty years,[208][209] and unemployment rising back to 7% in late 2009.[210] Unemployment rates for different age groups follow similar trends, but are consistently higher among youth. In the December 2014 quarter, the general unemployment rate was around 5.8%, while the unemployment rate for youth aged 15 to 21 was 15.6%.[207] New Zealand has experienced a series of "brain drains" since the 1970s[211] that still continue today.[212] Nearly one quarter of highly skilled workers live overseas, mostly in Australia and Britain, which is the largest proportion from any developed nation.[213] In recent decades, however, a "brain gain" has brought in educated professionals from Europe and less developed countries.[214][215] Today New Zealand's economy benefits from a high level of innovation.[216]

                    Trade

                    New Zealand is heavily dependent on international trade,[217] particularly in agricultural products.[218] Exports account for 24% of its output,[143] making New Zealand vulnerable to international commodity prices and global economic slowdowns. Food products made up 55% of the value of all the country's exports in 2014; wood was the second largest earner (7%).[219] New Zealand's main trading partners, as at June 2018, are China (NZ$27.8b), Australia ($26.2b), the European Union ($22.9b), the United States ($17.6b), and Japan ($8.4b).[220] On 7 April 2008, New Zealand and China signed the New Zealand–China Free Trade Agreement, the first such agreement China has signed with a developed country.[221] The service sector is the largest sector in the economy, followed by manufacturing and construction and then farming and raw material extraction.[143] Tourism plays a significant role in the economy, contributing $12.9 billion (or 5.6%) to New Zealand's total GDP and supporting 7.5% of the total workforce in 2016.[222] International visitor arrivals are expected to increase at a rate of 5.4% annually up to 2022.[222]

                    A Romney ewe with her two lambs

                    Wool has historically been one of New Zealand's major exports.

                    Wool was New Zealand's major agricultural export during the late 19th century.[196] Even as late as the 1960s it made up over a third of all export revenues,[196] but since then its price has steadily dropped relative to other commodities[223] and wool is no longer profitable for many farmers.[224] In contrast dairy farming increased, with the number of dairy cows doubling between 1990 and 2007,[225] to become New Zealand's largest export earner.[226] In the year to June 2018, dairy products accounted for 17.7% ($14.1 billion) of total exports,[220] and the country's largest company, Fonterra, controls almost one-third of the international dairy trade.[227] Other exports in 2017-18 were meat (8.8%), wood and wood products (6.2%), fruit (3.6%), machinery (2.2%) and wine (2.1%).[220] New Zealand's wine industry has followed a similar trend to dairy, the number of vineyards doubling over the same period,[228] overtaking wool exports for the first time in 2007.[229][230]

                    Infrastructure

                    A mid-size jet airliner in flight. The plane livery is all-black and features a New Zealand silver fern mark.

                    In 2015, renewable energy, primarily geothermal and hydroelectric power, generated 40.1% of New Zealand's gross energy supply.[231] Geothermal power alone accounted for 22% of New Zealand's energy in 2015.[231]

                    The provision of water supply and sanitation is generally of good quality. Regional authorities provide water abstraction, treatment and distribution infrastructure to most developed areas.[232][233]

                    New Zealand's transport network comprises 94,000 kilometres (58,410 mi) of roads, including 199 kilometres (124 mi) of motorways,[234] and 4,128 kilometres (2,565 mi) of railway lines.[143] Most major cities and towns are linked by bus services, although the private car is the predominant mode of transport.[235] The railways were privatised in 1993, but were re-nationalised by the government in stages between 2004 and 2008. The state-owned enterprise KiwiRail now operates the railways, with the exception of commuter services in Auckland and Wellington which are operated by Transdev[236] and Metlink,[237] respectively. Railways run the length of the country, although most lines now carry freight rather than passengers.[238] Most international visitors arrive via air[239] and New Zealand has six international airports, but currently only the Auckland and Christchurch airports connect directly with countries other than Australia or Fiji.[240]

                    The New Zealand Post Office had a monopoly over telecommunications in New Zealand until 1987 when Telecom New Zealand was formed, initially as a state-owned enterprise and then privatised in 1990.[241] Chorus, which was split from Telecom (now Spark) in 2011,[242] still owns the majority of the telecommunications infrastructure, but competition from other providers has increased.[241] A large-scale rollout of gigabit-capable fibre to the premises, branded as Ultra-Fast Broadband, began in 2009 with a target of being available to 87% of the population by 2022.[243] As of 2017, the United Nations International Telecommunication Union ranks New Zealand 13th in the development of information and communications infrastructure.[244]

                    Demography

                    Stationary population pyramid broken down into 21 age ranges.

                    The 2013 New Zealand census enumerated a resident population of 4,242,048, an increase of 5.3% over the 2006 figure.[245][n 8] As of September 2019, the total population has risen to an estimated 4,933,210.[5]

                    New Zealand is a predominantly urban country, with 73.0% of the population living in the seventeen main urban areas (i.e. population 30,000 or greater) and 55.1% living in the four largest cities of Auckland, Christchurch, Wellington, and Hamilton.[247] New Zealand cities generally rank highly on international livability measures. For instance, in 2016 Auckland was ranked the world's third most liveable city and Wellington the twelfth by the Mercer Quality of Living Survey.[248]

                    Life expectancy for New Zealanders in 2012 was 84 years for females, and 80.2 years for males.[249] Life expectancy at birth is forecast to increase from 80 years to 85 years in 2050 and infant mortality is expected to decline.[250] New Zealand's fertility rate of 2.1 is relatively high for a developed country, and natural births account for a significant proportion of population growth. Consequently, the country has a young population compared to most industrialised nations, with 20% of New Zealanders being 14 years old or younger.[143] In 2018 the median age of the New Zealand population was 38.1 years.[251] By 2050 the median age is projected to rise to 43 years and the percentage of people 60 years of age and older to rise from 18% to 29%.[250] In 2008 the leading cause of premature death was cancer, at 29.8%, followed by ischaemic heart disease, 19.7%, and then cerebrovascular disease, 9.2%.[252] As of 2016, total expenditure on health care (including private sector spending) is 9.2% of GDP.[253]

                    Largest urban areas in New Zealand

                    Statistics New Zealand June 2018 estimate (NZSAC92 boundaries)[254]

                    Rank Name Region Pop. Rank Name Region Pop.
                    Auckland
                    Auckland
                    Wellington
                    Wellington                     		1 Auckland Auckland 1,628,900 11 Whangarei Northland 58,800 Christchurch
                    Christchurch
                    Hamilton
                    Hamilton
                    2 Wellington Wellington 418,500 12 New Plymouth Taranaki 58,300
                    3 Christchurch Canterbury 404,500 13 Invercargill Southland 51,200
                    4 Hamilton Waikato 241,200 14 Kapiti Wellington 42,700
                    5 Tauranga Bay of Plenty 141,600 15 Whanganui Manawatu-Wanganui 40,900
                    6 Napier-Hastings Hawke's Bay 134,500 16 Gisborne Gisborne 37,200
                    7 Dunedin Otago 122,000 17 Blenheim Marlborough 31,600
                    8 Palmerston North Manawatu-Wanganui 86,600 18 Pukekohe Auckland 31,400
                    9 Nelson Nelson 67,500 19 Timaru Canterbury 29,100
                    10 Rotorua Bay of Plenty 59,500 20 Taupo Waikato 24,700

                    Ethnicity and immigration

                    Pedestrians crossing a wide street which is flanked by storefronts

                    Pedestrians on Queen Street in Auckland, an ethnically diverse city

                    In the 2013 census, 74.0% of New Zealand residents identified ethnically as European, and 14.9% as Māori. Other major ethnic groups include Asian (11.8%) and Pacific peoples (7.4%), two-thirds of whom live in the Auckland Region.[255][n 3] The population has become more diverse in recent decades: in 1961, the census reported that the population of New Zealand was 92% European and 7% Māori, with Asian and Pacific minorities sharing the remaining 1%.[256]

                    While the demonym for a New Zealand citizen is New Zealander, the informal "Kiwi" is commonly used both internationally[257] and by locals.[258] The Māori loanword Pākehā has been used to refer to New Zealanders of European descent, although others reject this appellation.[259][260] The word Pākehā today is increasingly used to refer to all non-Polynesian New Zealanders.[261]

                    The Māori were the first people to reach New Zealand, followed by the early European settlers. Following colonisation, immigrants were predominantly from Britain, Ireland and Australia because of restrictive policies similar to the White Australia policy.[262] There was also significant Dutch, Dalmatian,[263] German, and Italian immigration, together with indirect European immigration through Australia, North America, South America and South Africa.[264][265] Net migration increased after the Second World War; in the 1970s and 1980s policies were relaxed and immigration from Asia was promoted.[265][266] In 2009–10, an annual target of 45,000–50,000 permanent residence approvals was set by the New Zealand Immigration Service—more than one new migrant for every 100 New Zealand residents.[267] Just over 25% of New Zealand's population was born overseas, with the majority (52%) living in the Auckland Region. The United Kingdom remains the largest source of New Zealand's overseas population, with a quarter of all overseas-born New Zealanders born there; other major sources of New Zealand's overseas-born population are China, India, Australia, South Africa, Fiji and Samoa.[268] The number of fee-paying international students increased sharply in the late 1990s, with more than 20,000 studying in public tertiary institutions in 2002.[269]

                    Language

                    Map of New Zealand showing the percentage of people in each census area unit who speak Māori. Areas of the North Island exhibit the highest Māori proficiency.

                    Speakers of Māori according to the 2013 census[270]

                      Less than 5%

                      More than 5%

                      More than 10%

                      More than 20%

                      More than 30%

                      More than 40%

                      More than 50%

                    English is the predominant language in New Zealand, spoken by 96.1% of the population.[271] New Zealand English is similar to Australian English and many speakers from the Northern Hemisphere are unable to tell the accents apart.[272] The most prominent differences between the New Zealand English dialect and other English dialects are the shifts in the short front vowels: the short-"i" sound (as in "kit") has centralised towards the schwa sound (the "a" in "comma" and "about"); the short-"e" sound (as in "dress") has moved towards the short-"i" sound; and the short-"a" sound (as in "trap") has moved to the short-"e" sound.[273]

                    After the Second World War, Māori were discouraged from speaking their own language (te reo Māori) in schools and workplaces and it existed as a community language only in a few remote areas.[274] It has recently undergone a process of revitalisation,[275] being declared one of New Zealand's official languages in 1987,[276] and is spoken by 3.7% of the population.[271][n 9] There are now Māori language immersion schools and two television channels that broadcast predominantly in Māori.[278] Many places have both their Māori and English names officially recognised.[279]

                    As recorded in the 2013 census,[271] Samoan is the most widely spoken non-official language (2.2%),[n 10] followed by Hindi (1.7%), "Northern Chinese" (including Mandarin, 1.3%) and French (1.2%). 20,235 people (0.5%) reported the ability to use New Zealand Sign Language. It was declared one of New Zealand's official languages in 2006.[280]

                    Religion

                    Simple white building with two red domed towers

                    A Rātana church on a hill near Raetihi. The two-tower construction is characteristic of Rātana buildings.

                    Christianity is the predominant religion in New Zealand, although its society is among the most secular in the world.[281][282] In the 2018 census, 51.4% of the population identified with one or more religions, including 38.6% identifying as Christians. Another 48.6% indicated that they had no religion.[n 11] The main Christian denominations are, by number of adherents, Roman Catholicism (10.1%), Anglicanism (6.8%), Presbyterianism (5.5%) and "Christian not further defined" (i.e. people identifying as Christian but not stating the denomination, 6.6%). The Māori-based Ringatū and Rātana religions (1.3%) are also Christian in origin.[284][285] Immigration and demographic change in recent decades has contributed to the growth of minority religions,[286] such as Hinduism (2.6%), Buddhism (1.1%), Islam (1.3%) and Sikhism (0.5%).[284] The Auckland Region exhibited the greatest religious diversity.[284]

                    Education

                    Primary and secondary schooling is compulsory for children aged 6 to 16, with the majority attending from the age of 5.[287] There are 13 school years and attending state (public) schools is free to New Zealand citizens and permanent residents from a person's 5th birthday to the end of the calendar year following their 19th birthday.[288] New Zealand has an adult literacy rate of 99%,[143] and over half of the population aged 15 to 29 hold a tertiary qualification.[287] There are five types of government-owned tertiary institutions: universities, colleges of education, polytechnics, specialist colleges, and wānanga,[289] in addition to private training establishments.[290] In the adult population 14.2% have a bachelor's degree or higher, 30.4% have some form of secondary qualification as their highest qualification and 22.4% have no formal qualification.[291] The OECD's Programme for International Student Assessment ranks New Zealand's education system as the seventh best in the world, with students performing exceptionally well in reading, mathematics and science.[292]

                    Culture

                    Tall wooden carving showing Kupe above two tentacled sea creatures

                    Late 20th-century house-post depicting the navigator Kupe fighting two sea creatures

                    Early Māori adapted the tropically based east Polynesian culture in line with the challenges associated with a larger and more diverse environment, eventually developing their own distinctive culture. Social organisation was largely communal with families (whānau), subtribes (hapū) and tribes (iwi) ruled by a chief (rangatira), whose position was subject to the community's approval.[293] The British and Irish immigrants brought aspects of their own culture to New Zealand and also influenced Māori culture,[294][295] particularly with the introduction of Christianity.[296] However, Māori still regard their allegiance to tribal groups as a vital part of their identity, and Māori kinship roles resemble those of other Polynesian peoples.[297] More recently American, Australian, Asian and other European cultures have exerted influence on New Zealand. Non-Māori Polynesian cultures are also apparent, with Pasifika, the world's largest Polynesian festival, now an annual event in Auckland.[298]

                    The largely rural life in early New Zealand led to the image of New Zealanders being rugged, industrious problem solvers.[299] Modesty was expected and enforced through the "tall poppy syndrome", where high achievers received harsh criticism.[300] At the time New Zealand was not known as an intellectual country.[301] From the early 20th century until the late 1960s, Māori culture was suppressed by the attempted assimilation of Māori into British New Zealanders.[274] In the 1960s, as tertiary education became more available and cities expanded[302] urban culture began to dominate.[303] However, rural imagery and themes are common in New Zealand's art, literature and media.[304]

                    New Zealand's national symbols are influenced by natural, historical, and Māori sources. The silver fern is an emblem appearing on army insignia and sporting team uniforms.[305] Certain items of popular culture thought to be unique to New Zealand are called "Kiwiana".[305]

                    Art

                    As part of the resurgence of Māori culture, the traditional crafts of carving and weaving are now more widely practised and Māori artists are increasing in number and influence.[306] Most Māori carvings feature human figures, generally with three fingers and either a natural-looking, detailed head or a grotesque head.[307] Surface patterns consisting of spirals, ridges, notches and fish scales decorate most carvings.[308] The pre-eminent Māori architecture consisted of carved meeting houses (wharenui) decorated with symbolic carvings and illustrations. These buildings were originally designed to be constantly rebuilt, changing and adapting to different whims or needs.[309]

                    Māori decorated the white wood of buildings, canoes and cenotaphs using red (a mixture of red ochre and shark fat) and black (made from soot) paint and painted pictures of birds, reptiles and other designs on cave walls.[310] Māori tattoos (moko) consisting of coloured soot mixed with gum were cut into the flesh with a bone chisel.[311] Since European arrival paintings and photographs have been dominated by landscapes, originally not as works of art but as factual portrayals of New Zealand.[312] Portraits of Māori were also common, with early painters often portraying them as "noble savages", exotic beauties or friendly natives.[312] The country's isolation delayed the influence of European artistic trends allowing local artists to develop their own distinctive style of regionalism.[313] During the 1960s and 1970s many artists combined traditional Māori and Western techniques, creating unique art forms.[314] New Zealand art and craft has gradually achieved an international audience, with exhibitions in the Venice Biennale in 2001 and the "Paradise Now" exhibition in New York in 2004.[306][315]

                    Refer to caption

                    Māori cloaks are made of fine flax fibre and patterned with black, red and white triangles, diamonds and other geometric shapes.[316] Greenstone was fashioned into earrings and necklaces, with the most well-known design being the hei-tiki, a distorted human figure sitting cross-legged with its head tilted to the side.[317] Europeans brought English fashion etiquette to New Zealand, and until the 1950s most people dressed up for social occasions.[318] Standards have since relaxed and New Zealand fashion has received a reputation for being casual, practical and lacklustre.[319][320] However, the local fashion industry has grown significantly since 2000, doubling exports and increasing from a handful to about 50 established labels, with some labels gaining international recognition.[320]

                    Literature

                    Māori quickly adopted writing as a means of sharing ideas, and many of their oral stories and poems were converted to the written form.[321] Most early English literature was obtained from Britain and it was not until the 1950s when local publishing outlets increased that New Zealand literature started to become widely known.[322] Although still largely influenced by global trends (modernism) and events (the Great Depression), writers in the 1930s began to develop stories increasingly focused on their experiences in New Zealand. During this period literature changed from a journalistic activity to a more academic pursuit.[323] Participation in the world wars gave some New Zealand writers a new perspective on New Zealand culture and with the post-war expansion of universities local literature flourished.[324] Dunedin is a UNESCO City of Literature.[325]

                    Media and entertainment

                    New Zealand music has been influenced by blues, jazz, country, rock and roll and hip hop, with many of these genres given a unique New Zealand interpretation.[326] Māori developed traditional chants and songs from their ancient Southeast Asian origins, and after centuries of isolation created a unique "monotonous" and "doleful" sound.[327] Flutes and trumpets were used as musical instruments[328] or as signalling devices during war or special occasions.[329] Early settlers brought over their ethnic music, with brass bands and choral music being popular, and musicians began touring New Zealand in the 1860s.[330][331] Pipe bands became widespread during the early 20th century.[332] The New Zealand recording industry began to develop from 1940 onwards and many New Zealand musicians have obtained success in Britain and the United States.[326] Some artists release Māori language songs and the Māori tradition-based art of kapa haka (song and dance) has made a resurgence.[333] The New Zealand Music Awards are held annually by Recorded Music NZ; the awards were first held in 1965 by Reckitt & Colman as the Loxene Golden Disc awards.[334] Recorded Music NZ also publishes the country's official weekly record charts.[335]

                    Hills with inset, round doors. Reflected in water.

                    Public radio was introduced in New Zealand in 1922.[337] A state-owned television service began in 1960.[338] Deregulation in the 1980s saw a sudden increase in the numbers of radio and television stations.[339] New Zealand television primarily broadcasts American and British programming, along with a large number of Australian and local shows.[340] The number of New Zealand films significantly increased during the 1970s. In 1978 the New Zealand Film Commission started assisting local film-makers and many films attained a world audience, some receiving international acknowledgement.[339] The highest-grossing New Zealand films are Hunt for the Wilderpeople, Boy, The World's Fastest Indian, Once Were Warriors and Whale Rider.[341] The country's diverse scenery and compact size, plus government incentives,[342] have encouraged some producers to shoot big-budget productions in New Zealand, including Avatar, The Lord of the Rings, The Hobbit, The Chronicles of Narnia, King Kong and The Last Samurai.[343] The New Zealand media industry is dominated by a small number of companies, most of which are foreign-owned, although the state retains ownership of some television and radio stations.[344] Since 1994, Freedom House has consistently ranked New Zealand's press freedom in the top twenty, with the 19th freest media in 2015.[345]

                    Sports

                    Rugby team wearing all black, facing the camera, knees bent, and facing toward a team wearing white

                    Most of the major sporting codes played in New Zealand have British origins.[346] Rugby union is considered the national sport[347] and attracts the most spectators.[348] Golf, netball, tennis and cricket have the highest rates of adult participation, while netball, rugby union and football (soccer) are particularly popular among young people.[348][349] Around 54% of New Zealand adolescents participate in sports for their school.[349] Victorious rugby tours to Australia and the United Kingdom in the late 1880s and the early 1900s played an early role in instilling a national identity.[350] Horseracing was also a popular spectator sport and became part of the "Rugby, Racing and Beer" culture during the 1960s.[351] Māori participation in European sports was particularly evident in rugby and the country's team performs a haka, a traditional Māori challenge, before international matches.[352] New Zealand is known for its extreme sports, adventure tourism[353] and strong mountaineering tradition, as seen in the success of notable New Zealander Sir Edmund Hillary.[354][355] Other outdoor pursuits such as cycling, fishing, swimming, running, tramping, canoeing, hunting, snowsports, surfing and sailing are also popular.[356] The Polynesian sport of waka ama racing has experienced a resurgence of interest in New Zealand since the 1980s.[357]

                    New Zealand has competitive international teams in rugby union, rugby league, netball, cricket, softball, and sailing. New Zealand participated at the Summer Olympics in 1908 and 1912 as a joint team with Australia, before first participating on its own in 1920.[358] The country has ranked highly on a medals-to-population ratio at recent Games.[359][360] The "All Blacks", the national rugby union team, are the most successful in the history of international rugby[361] and the reigning World Cup champions.[362]

                    Cuisine

                    Raw meat and vegetables

                    Ingredients to be prepared for a hāngi

                    The national cuisine has been described as Pacific Rim, incorporating the native Māori cuisine and diverse culinary traditions introduced by settlers and immigrants from Europe, Polynesia and Asia.[363] New Zealand yields produce from land and sea—most crops and livestock, such as maize, potatoes and pigs, were gradually introduced by the early European settlers.[364] Distinctive ingredients or dishes include lamb, salmon, kōura (crayfish),[365] dredge oysters, whitebait, pāua (abalone), mussels, scallops, pipis and tuatua (both are types of New Zealand shellfish),[366] kūmara (sweet potato), kiwifruit, tamarillo and pavlova (considered a national dish).[367][363] A hāngi is a traditional Māori method of cooking food using heated rocks buried in a pit oven. After European colonisation, Māori began cooking with pots and ovens and the hāngi was used less frequently, although it is still used for formal occasions such as tangihanga.[368]

                    See also

                    Footnotes

                    1. ^ "God Save the Queen" is officially a national anthem but is generally used only on regal and viceregal occasions.[1]
                    2. ^ English is a de facto official language due to its widespread use.[2]
                    3. ^ a b Ethnicity figures add to more than 100% as people could choose more than one ethnic group.
                    4. ^ The proportion of New Zealand's area (excluding estuaries) covered by rivers, lakes and ponds, based on figures from the New Zealand Land Cover Database,[4] is (357526 + 81936) / (26821559 – 92499–26033 – 19216) = 1.6%. If estuarine open water, mangroves, and herbaceous saline vegetation are included, the figure is 2.2%.
                    5. ^ The Chatham Islands have a separate time zone, 45 minutes ahead of the rest of New Zealand.
                    6. ^ Clocks are advanced by an hour from the last Sunday in September until the first Sunday in April.[9] Daylight saving time is also observed in the Chatham Islands, 45 minutes ahead of NZDT.
                    7. ^ A person born on or after 1 January 2006 acquires New Zealand citizenship at birth only if at least one parent is a New Zealand citizen or permanent resident. People born on or before 31 December 2005 acquired citizenship at birth (jus soli).[137]
                    8. ^ The population is increasing at a rate of 1.4–2.0% per year and is projected to rise to 5.01–5.51 million in 2025.[246]
                    9. ^ In 2015, 55% of Māori adults (aged 15 years and over) reported knowledge of te reo Māori. Of these speakers, 64% use Māori at home and 50,000 can speak the language "very well" or "well".[277]
                    10. ^ Of the 86,403 people that replied they spoke Samoan, 51,336 lived in the Auckland Region.
                    11. ^ Religion percentages may not add to 100% as people could claim multiple religions or object to answering the question.

                    Citations

                    1. ^ "Protocol for using New Zealand's National Anthems". Ministry for Culture and Heritage. Retrieved 17 February 2008.
                    2. ^ New Zealand Government (21 December 2007). International Covenant on Civil and Political Rights Fifth Periodic Report of the Government of New Zealand (PDF) (Report). p. 89. Archived from the original (PDF) on 24 January 2015. Retrieved 18 November 2015. In addition to the Māori language, New Zealand Sign Language is also an official language of New Zealand. The New Zealand Sign Language Act 2006 permits the use of NZSL in legal proceedings, facilitates competency standards for its interpretation and guides government departments in its promotion and use. English, the medium for teaching and learning in most schools, is a de facto official language by virtue of its widespread use. For these reasons, these three languages have special mention in the New Zealand Curriculum.
                    3. ^ "2018 Census population and dwelling counts". Statistics New Zealand. Retrieved 26 September 2019.
                    4. ^ "The New Zealand Land Cover Database". New Zealand Land Cover Database 2. Ministry for the Environment. 1 July 2009. Retrieved 26 April 2011.
                    5. ^ a b "Population clock". Statistics New Zealand. Retrieved 14 April 2016. The population estimate shown is automatically calculated daily at 00:00 UTC and is based on data obtained from the population clock on the date shown in the citation.
                    6. ^ a b c d e "New Zealand". International Monetary Fund. Retrieved 9 October 2018.
                    7. ^ "Income inequality". Statistics New Zealand. Retrieved 14 June 2015.
                    8. ^ a b "Human Development Report 2018" (PDF). HDRO (Human Development Report Office) United Nations Development Programme. p. 22. Retrieved 14 September 2018.
                    9. ^ "New Zealand Daylight Time Order 2007 (SR 2007/185)". New Zealand Parliamentary Counsel Office. 6 July 2007. Retrieved 6 March 2017.
                    10. ^ There is no official all-numeric date format for New Zealand, but government recommendations generally follow Australian date and time notation. See "The Govt.nz style guide", New Zealand Government, 9 December 2016, retrieved 7 March 2019 .
                    11. ^ Tasman, Abel. "JOURNAL or DESCRIPTION By me Abel Jansz Tasman, Of a Voyage from Batavia for making Discoveries of the Unknown South Land in the year 1642". Project Gutenberg Australia. Retrieved 26 March 2018.
                    12. ^ Wilson, John (March 2009). "European discovery of New Zealand – Tasman's achievement". Te Ara: The Encyclopedia of New Zealand. Retrieved 24 January 2011.
                    13. ^ John Bathgate. "The Pamphlet Collection of Sir Robert Stout:Volume 44. Chapter 1, Discovery and Settlement". NZETC. Retrieved 17 August 2018. He named the country Staaten Land, in honour of the States-General of Holland, in the belief that it was part of the great southern continent.
                    14. ^ Wilson, John (September 2007). "Tasman's achievement". Te Ara: The Encyclopedia of New Zealand. Retrieved 16 February 2008.
                    15. ^ Mackay, Duncan (1986). "The Search For The Southern Land". In Fraser, B (ed.). The New Zealand Book Of Events. Auckland: Reed Methuen. pp. 52–54.
                    16. ^ a b McKinnon, Malcolm (November 2009). "Place names – Naming the country and the main islands". Te Ara: The Encyclopedia of New Zealand. Retrieved 24 January 2011.
                    17. ^ King 2003, p. 41.
                    18. ^ Hay, Maclagan & Gordon 2008, p. 72.
                    19. ^ a b Mein Smith 2005, p. 6.
                    20. ^ Brunner, Thomas (1851). The Great Journey: an expedition to explore the interior of the Middle Island, New Zealand, 1846-8. Royal Geographical Society.
                    21. ^ a b Williamson, Maurice (10 October 2013). "Names of NZ's two main islands formalised" (Press release). New Zealand Government. Retrieved 1 May 2017.
                    22. ^ Wilmshurst, J. M.; Hunt, T. L.; Lipo, C. P.; Anderson, A. J. (2010). "High-precision radiocarbon dating shows recent and rapid initial human colonization of East Polynesia". Proceedings of the National Academy of Sciences. 108 (5): 1815. Bibcode:2011PNAS..108.1815W. doi:10.1073/pnas.1015876108. PMC 3033267. PMID 21187404.
                    23. ^ McGlone, M.; Wilmshurst, J. M. (1999). "Dating initial Maori environmental impact in New Zealand". Quaternary International. 59: 5–16. Bibcode:1999QuInt..59....5M. doi:10.1016/S1040-6182(98)00067-6.
                    24. ^ Murray-McIntosh, Rosalind P.; Scrimshaw, Brian J.; Hatfield, Peter J.; Penny, David (1998). "Testing migration patterns and estimating founding population size in Polynesia by using human mtDNA sequences". Proceedings of the National Academy of Sciences of the United States of America. 95 (15): 9047–52. Bibcode:1998PNAS...95.9047M. doi:10.1073/pnas.95.15.9047. PMC 21200.
                    25. ^ Wilmshurst, J. M.; Anderson, A. J.; Higham, T. F. G.; Worthy, T. H. (2008). "Dating the late prehistoric dispersal of Polynesians to New Zealand using the commensal Pacific rat". Proceedings of the National Academy of Sciences. 105 (22): 7676. Bibcode:2008PNAS..105.7676W. doi:10.1073/pnas.0801507105. PMC 2409139. PMID 18523023.
                    26. ^ Moodley, Y.; Linz, B.; Yamaoka, Y.; Windsor, H.M.; Breurec, S.; Wu, J.-Y.; Maady, A.; Bernhöft, S.; Thiberge, J.-M.; et al. (2009). "The Peopling of the Pacific from a Bacterial Perspective". Science. 323 (5913): 527–530. Bibcode:2009Sci...323..527M. doi:10.1126/science.1166083. PMC 2827536. PMID 19164753.
                    27. ^ Ballara, Angela (1998). Iwi: The Dynamics of Māori Tribal Organisation from c. 1769 to c. 1945 (1st ed.). Wellington: Victoria University Press. ISBN 9780864733283.
                    28. ^ Clark, Ross (1994). "Moriori and Māori: The Linguistic Evidence". In Sutton, Douglas (ed.). The Origins of the First New Zealanders. Auckland: Auckland University Press. pp. 123–135.
                    29. ^ Davis, Denise (September 2007). "The impact of new arrivals". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 April 2010.
                    30. ^ Davis, Denise; Solomon, Māui (March 2009). "'Moriori – The impact of new arrivals'". Te Ara: The Encyclopedia of New Zealand. Retrieved 23 March 2011.
                    31. ^ a b Mein Smith 2005, p. 23.
                    32. ^ Salmond, Anne. Two Worlds: First Meetings Between Maori and Europeans 1642–1772. Auckland: Penguin Books. p. 82. ISBN 0-670-83298-7.
                    33. ^ King 2003, p. 122.
                    34. ^ Fitzpatrick, John (2004). "Food, warfare and the impact of Atlantic capitalism in Aotearo/New Zealand" (PDF). Australasian Political Studies Association Conference: APSA 2004 Conference Papers. Archived from the original (PDF) on 11 May 2011.
                    35. ^ Brailsford, Barry (1972). Arrows of Plague. Wellington: Hick Smith and Sons. p. 35. ISBN 0-456-01060-2.
                    36. ^ Wagstrom, Thor (2005). "Broken Tongues and Foreign Hearts". In Brock, Peggy (ed.). Indigenous Peoples and Religious Change. Boston: Brill Academic Publishers. pp. 71 and 73. ISBN 978-90-04-13899-5.
                    37. ^ Lange, Raeburn (1999). May the people live: a history of Māori health development 1900–1920. Auckland University Press. p. 18. ISBN 978-1-86940-214-3.
                    38. ^ "A Nation sub-divided". Australian Heritage. Heritage Australia Publishing. 2011. Archived from the original on 28 February 2015. Retrieved 27 December 2014.
                    39. ^ a b Rutherford, James (April 2009) [originally published in 1966]. McLintock, Alexander (ed.). Busby, James. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    40. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Sir George Gipps. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    41. ^ a b Wilson, John (March 2009). "Government and nation – The origins of nationhood". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    42. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Settlement from 1840 to 1852. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    43. ^ Foster, Bernard (April 2009) [originally published in 1966]. McLintock, Alexander (ed.). Akaroa, French Settlement At. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    44. ^ Simpson, K (September 2010). "Hobson, William – Biography". In McLintock, Alexander (ed.). Dictionary of New Zealand Biography. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    45. ^ Phillips, Jock (April 2010). "British immigration and the New Zealand Company". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    46. ^ "Crown colony era – the Governor-General". Ministry for Culture and Heritage. March 2009. Retrieved 7 January 2011.
                    47. ^ "New Zealand's 19th-century wars – overview". Ministry for Culture and Heritage. April 2009. Retrieved 7 January 2011.
                    48. ^ a b c d Wilson, John (March 2009). "Government and nation – The constitution". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 February 2011. See pages 2 and 3.
                    49. ^ Temple, Philip (1980). Wellington Yesterday. John McIndoe. ISBN 0-86868-012-5.
                    50. ^ "Parliament moves to Wellington". Ministry for Culture and Heritage. January 2017. Retrieved 27 April 2017.
                    51. ^ a b Wilson, John (March 2009). "History – Liberal to Labour". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 April 2017.
                    52. ^ Hamer, David. "Seddon, Richard John". Dictionary of New Zealand Biography. Ministry for Culture and Heritage. Retrieved 27 April 2017.
                    53. ^ Boxall, Peter; Haynes, Peter (1997). "Strategy and Trade Union Effectiveness in a Neo-liberal Environment". British Journal of Industrial Relations. 35 (4): 567–591. doi:10.1111/1467-8543.00069. Archived from the original (PDF) on 11 May 2011.
                    54. ^ "Proclamation". The London Gazette. No. 28058. 10 September 1907. p. 6149.
                    55. ^ "Dominion status – Becoming a dominion". Ministry for Culture and Heritage. September 2014. Retrieved 26 April 2017.
                    56. ^ "War and Society". Ministry for Culture and Heritage. Retrieved 7 January 2011.
                    57. ^ Easton, Brian (April 2010). "Economic history – Interwar years and the great depression". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    58. ^ Derby, Mark (May 2010). "Strikes and labour disputes – Wars, depression and first Labour government". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    59. ^ Easton, Brian (November 2010). "Economic history – Great boom, 1935–1966". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    60. ^ Keane, Basil (November 2010). "Te Māori i te ohanga – Māori in the economy – Urbanisation". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    61. ^ Royal, Te Ahukaramū (March 2009). "Māori – Urbanisation and renaissance". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    62. ^ Healing the past, building a future: A Guide to Treaty of Waitangi Claims and Negotiations with the Crown (PDF). Office of Treaty Settlements. March 2015. ISBN 978-0-478-32436-5. Retrieved 26 April 2017.
                    63. ^ Report on the Crown's Foreshore and Seabed Policy (Report). Ministry of Justice. Retrieved 26 April 2017.
                    64. ^ Barker, Fiona (June 2012). "Debate about the foreshore and seabed". Te Ara: The Encyclopedia of New Zealand. Retrieved 26 April 2017.
                    65. ^ a b "New Zealand's Constitution". The Governor-General of New Zealand. Retrieved 13 January 2010.
                    66. ^ a b c "Factsheet – New Zealand – Political Forces". The Economist. The Economist Group. 15 February 2005. Archived from the original on 14 May 2006. Retrieved 4 August 2009.
                    67. ^ "Royal Titles Act 1974". New Zealand Parliamentary Counsel Office. February 1974. Section 1. Retrieved 8 January 2011.
                    68. ^ "Constitution Act 1986". New Zealand Parliamentary Counsel Office. 1 January 1987. Section 2.1. Retrieved 15 July 2018. The Sovereign in right of New Zealand is the head of State of New Zealand, and shall be known by the royal style and titles proclaimed from time to time.
                    69. ^ "The Role of the Governor-General". The Governor-General of New Zealand. Retrieved 6 July 2017.
                    70. ^ Harris, Bruce (2009). "Replacement of the Royal Prerogative in New Zealand". New Zealand Universities Law Review. 23: 285–314. Archived from the original on 18 July 2011. Retrieved 28 August 2016.
                    71. ^ a b "The Reserve Powers". The Governor-General of New Zealand. Retrieved 8 January 2011.
                    72. ^ a b c d "Parliament Brief: What is Parliament?". New Zealand Parliament. Retrieved 30 November 2016.
                    73. ^ McLean, Gavin (February 2015). "Premiers and prime ministers". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 November 2016.
                    74. ^ Wilson, John (November 2010). "Government and nation – System of government". Te Ara: The Encyclopedia of New Zealand. Retrieved 9 January 2011.
                    75. ^ "Principles of Cabinet decision making". Cabinet Manual. Department of the Prime Minister and Cabinet. 2008. Retrieved 1 December 2016.
                    76. ^ "The electoral cycle". Cabinet Manual. Department of the Prime Minister and Cabinet. 2008. Retrieved 30 April 2017.
                    77. ^ a b "First past the post – the road to MMP". Ministry for Culture and Heritage. September 2009. Retrieved 9 January 2011.
                    78. ^ "Reviewing electorate numbers and boundaries". Electoral Commission. 8 May 2005. Archived from the original on 9 November 2011. Retrieved 7 July 2018.
                    79. ^ "Sainte-Laguë allocation formula". Electoral Commission. 4 February 2013. Retrieved 31 May 2014.
                    80. ^ Paxton, Pamela; Hughes, Melanie M. (2015). Women, Politics, and Power: A Global Perspective. CQ Press. p. 107. ISBN 978-1-48-337701-8. Retrieved 25 July 2017.
                    81. ^ "Jacinda Ardern sworn in as new Prime Minister". New Zealand Herald. 26 October 2017. Retrieved 26 October 2017.
                    82. ^ "Female political leaders have been smashing glass ceilings for ages". Stuff.co.nz. Fairfax NZ. 27 October 2017. Retrieved 19 July 2018.
                    83. ^ "Role of the Chief Justice". Courts of New Zealand. Retrieved 9 June 2018.
                    84. ^ "Structure of the court system". Courts of New Zealand. Retrieved 9 June 2018.
                    85. ^ "The Judiciary". Ministry of Justice. Archived from the original on 24 November 2010. Retrieved 9 January 2011.
                    86. ^ "The Fragile States Index 2016". The Fund for Peace. Archived from the original on 4 February 2017. Retrieved 30 November 2016.
                    87. ^ "Democracy Index 2017" (PDF). Economist Intelligence Unit. 2018. p. 5. Retrieved 9 December 2018.
                    88. ^ "Corruption Perceptions Index 2017". Transparency International. 21 February 2018. Retrieved 9 December 2018.
                    89. ^ "New Zealand". Country Reports on Human Rights Practices for 2017. United States Department of State. Retrieved 9 December 2018.
                    90. ^ "New Zealand". OECD Better Life Index. 2016. Retrieved 30 November 2016.
                    91. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. External Relations. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    92. ^ "Michael Joseph Savage". Ministry for Culture and Heritage. July 2010. Retrieved 29 January 2011.
                    93. ^ Patman, Robert (2005). "Globalisation, Sovereignty, and the Transformation of New Zealand Foreign Policy" (PDF). Working Paper 21/05. Centre for Strategic Studies, Victoria University of Wellington. p. 8. Archived from the original (PDF) on 25 September 2007. Retrieved 12 March 2007.
                    94. ^ "Department Of External Affairs: Security Treaty between Australia, New Zealand and the United States of America". Australian Government. September 1951. Archived from the original on 29 June 2011. Retrieved 11 January 2011.
                    95. ^ "The Vietnam War". New Zealand History. Ministry for Culture and Heritage. June 2008. Retrieved 11 January 2011.
                    96. ^ "Sinking the Rainbow Warrior – nuclear-free New Zealand". New Zealand History. Ministry for Culture and Heritage. August 2008. Retrieved 11 January 2011.
                    97. ^ "Nuclear-free legislation – nuclear-free New Zealand". New Zealand History. Ministry for Culture and Heritage. August 2008. Retrieved 11 January 2011.
                    98. ^ Lange, David (1990). Nuclear Free: The New Zealand Way. New Zealand: Penguin Books. ISBN 0-14-014519-2.
                    99. ^ "Australia in brief". Australian Department of Foreign Affairs and Trade. Archived from the original on 22 December 2010. Retrieved 11 January 2011.
                    100. ^ a b "New Zealand country brief". Australian Department of Foreign Affairs and Trade. Retrieved 11 January 2011.
                    101. ^ Collett, John (4 September 2013). "Kiwis face hurdles in pursuit of lost funds". Retrieved 4 October 2013.
                    102. ^ Bertram, Geoff (April 2010). "South Pacific economic relations – Aid, remittances and tourism". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 January 2011.
                    103. ^ Howes, Stephen (November 2010). "Making migration work: Lessons from New Zealand". Development Policy Centre. Retrieved 23 March 2011.
                    104. ^ Ayele, Yoseph (28 September 2017). "The Growing Momentum for Global Impact in New Zealand". Edmund Hillary Fellowship. Retrieved 9 July 2019.
                    105. ^ "Member States of the United Nations". United Nations. Retrieved 11 January 2011.
                    106. ^ "New Zealand". The Commonwealth. Retrieved 1 December 2016.
                    107. ^ "Members and partners". Organisation for Economic Co-operation and Development. Retrieved 11 January 2011.
                    108. ^ "The future of the Five Power Defence Arrangements". The Strategist. 8 November 2012. Retrieved 1 December 2016.
                    109. ^ "About Us: Role and Responsibilities". New Zealand Defence Force. Retrieved 11 January 2011.
                    110. ^ Ayson, Robert (2007). "New Zealand Defence and Security Policy,1990–2005". In Alley, Roderic (ed.). New Zealand In World Affairs, Volume IV: 1990–2005. Wellington: Victoria University Press. p. 132. ISBN 978-0-86473-548-5.
                    111. ^ "The Battle for Crete". New Zealand History. Ministry for Culture and Heritage. May 2010. Retrieved 9 January 2011.
                    112. ^ "El Alamein – The North African Campaign". New Zealand History. Ministry for Culture and Heritage. May 2009. Retrieved 9 January 2011.
                    113. ^ Holmes, Richard (September 2010). "World War Two: The Battle of Monte Cassino". Retrieved 9 January 2011.
                    114. ^ "Gallipoli stirred new sense of national identity says Clark". New Zealand Herald. April 2005. Retrieved 9 January 2011.
                    115. ^ Prideaux, Bruce (2007). Ryan, Chris (ed.). Battlefield tourism: history, place and interpretation. Elsevier Science. p. 18. ISBN 978-0-08-045362-0.
                    116. ^ Burke, Arthur. "The Spirit of ANZAC". ANZAC Day Commemoration Committee. Archived from the original on 26 December 2010. Retrieved 11 January 2011.
                    117. ^ "South African War 1899–1902". Ministry for Culture and Heritage. February 2009. Retrieved 11 January 2011.
                    118. ^ "New Zealand in the Korean War". New Zealand History. Ministry for Culture and Heritage. Retrieved 1 December 2016.
                    119. ^ "NZ and the Malayan Emergency". Ministry for Culture and Heritage. August 2010. Retrieved 11 January 2011.
                    120. ^ "New Zealand Defence Force Overseas Operations". New Zealand Defence Force. January 2008. Archived from the original on 25 January 2008. Retrieved 17 February 2008.
                    121. ^ a b "New Zealand's Nine Provinces (1853–76)" (PDF). Friends of the Hocken Collections. University of Otago. March 2000. Retrieved 13 January 2011.
                    122. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Provincial Divergencies. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    123. ^ Swarbrick, Nancy (September 2016). "Public holidays". Te Ara: The Encyclopedia of New Zealand. Retrieved 25 June 2017.
                    124. ^ "Overview – regional rugby". Ministry for Culture and Heritage. September 2010. Retrieved 13 January 2011.
                    125. ^ Dollery, Brian; Keogh, Ciaran; Crase, Lin (2007). "Alternatives to Amalgamation in Australian Local Government: Lessons from the New Zealand Experience" (PDF). Sustaining Regions. 6 (1): 50–69. Archived from the original (PDF) on 29 August 2007.
                    126. ^ a b c Sancton, Andrew (2000). Merger mania: the assault on local government. McGill-Queen's University Press. p. 84. ISBN 0-7735-2163-1.
                    127. ^ "Subnational population estimates at 30 June 2010 (boundaries at 1 November 2010)". Statistics New Zealand. 26 October 2010. Archived from the original on 10 June 2011. Retrieved 2 April 2011.
                    128. ^ Smelt & Jui Lin 2009, p. 33.
                    129. ^ a b "Glossary". Department of Internal Affairs. Retrieved 28 August 2016.
                    130. ^ "Chatham Islands Council Act 1995 No 41". New Zealand Parliamentary Counsel Office. 29 July 1995. Retrieved 8 August 2017.
                    131. ^ Gimpel, Diane (2011). Monarchies. ABDO Publishing Company. p. 22. ISBN 978-1-617-14792-0. Retrieved 18 November 2016.
                    132. ^ "System of Government". Government of Niue. Archived from the original on 13 November 2010. Retrieved 13 January 2010.
                    133. ^ "Government – Structure, Personnel". Government of the Cook Islands. Retrieved 13 January 2010.
                    134. ^ "Tokelau Government". Government of Tokelau. Retrieved 16 November 2016.
                    135. ^ "Scott Base". Antarctica New Zealand. Retrieved 13 January 2010.
                    136. ^ "Citizenship Act 1977 No 61". Zealand Parliamentary Counsel Office. 1 December 1977. Retrieved 26 May 2017.
                    137. ^ "Check if you're a New Zealand citizen". Department of Internal Affairs. Retrieved 20 January 2015.
                    138. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. The Sea Floor. An Encyclopaedia of New Zealand. Retrieved 13 January 2011.
                    139. ^ "Hauraki Gulf islands". Auckland City Council. Archived from the original on 25 December 2010. Retrieved 13 January 2011.
                    140. ^ Hindmarsh (2006). "Discovering D'Urville". Heritage New Zealand. Archived from the original on 11 May 2011. Retrieved 13 January 2011.
                    141. ^ "Distance tables". Auckland Coastguard. Archived from the original on 23 January 2011. Retrieved 2 March 2011.
                    142. ^ McKenzie, D. W. (1987). Heinemann New Zealand atlas. Heinemann Publishers. ISBN 0-7900-0187-X.
                    143. ^ a b c d e f "CIA – The World Factbook". Cia.gov. Retrieved 4 May 2013.
                    144. ^ "Geography". Statistics New Zealand. 1999. Archived from the original on 22 May 2010. Retrieved 21 December 2009.
                    145. ^ Offshore Options: Managing Environmental Effects in New Zealand's Exclusive Economic Zone (PDF). Wellington: Ministry for the Environment. 2005. ISBN 0-478-25916-6. Retrieved 23 June 2017.
                    146. ^ Coates, Glen (2002). The rise and fall of the Southern Alps. Canterbury University Press. p. 15. ISBN 0-908812-93-0.
                    147. ^ Garden 2005, p. 52.
                    148. ^ Grant, David (March 2009). "Southland places – Fiordland's coast". Te Ara: The Encyclopedia of New Zealand. Retrieved 14 January 2011.
                    149. ^ "Central North Island volcanoes". Department of Conservation. Archived from the original on 29 December 2010. Retrieved 14 January 2011.
                    150. ^ Walrond, Carl (March 2009). "Natural environment – Geography and geology". Te Ara: The Encyclopedia of New Zealand. Retrieved 14 January 2010.
                    151. ^ "Taupo". GNS Science. Archived from the original on 24 March 2011. Retrieved 2 April 2011.
                    152. ^ a b Lewis, Keith; Nodder, Scott; Carter, Lionel (March 2009). "Sea floor geology – Active plate boundaries". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    153. ^ Wallis, G. P.; Trewick, S. A. (2009). "New Zealand phylogeography: evolution on a small continent". Molecular Ecology. 18 (17): 3548–3580. doi:10.1111/j.1365-294X.2009.04294.x. PMID 19674312.
                    154. ^ Wright, Dawn; Bloomer, Sherman; MacLeod, Christopher; Taylor, Brian; Goodliffe, Andrew (2000). "Bathymetry of the Tonga Trench and Forearc: A Map Series". Marine Geophysical Researches. 21 (5): 489–512. Bibcode:2000MarGR..21..489W. doi:10.1023/A:1026514914220.
                    155. ^ "Australasia". New Zealand Oxford Dictionary. Oxford University Press. 2005. doi:10.1093/acref/9780195584516.001.0001. ISBN 9780195584516.
                    156. ^ Hobbs, Joseph J. (2016). Fundamentals of World Regional Geography. Cengage Learning. p. 367. ISBN 9781305854956.
                    157. ^ Hillstrom, Kevin; Hillstrom, Laurie Collier (2003). Australia, Oceania, and Antarctica: A Continental Overview of Environmental Issues. 3. ABC-CLIO. p. 25. ISBN 9781576076941. …defined here as the continent nation of Australia, New Zealand, and twenty-two other island countries and territories sprinkled over more than 40 million square kilometres of the South Pacific.
                    158. ^ a b Mullan, Brett; Tait, Andrew; Thompson, Craig (March 2009). "Climate – New Zealand's climate". Te Ara: The Encyclopedia of New Zealand. Retrieved 15 January 2011.
                    159. ^ "Summary of New Zealand climate extremes". National Institute of Water and Atmospheric Research. 2004. Retrieved 30 April 2010.
                    160. ^ Walrond, Carl (March 2009). "Natural environment – Climate". Te Ara: The Encyclopedia of New Zealand. Retrieved 15 January 2011.
                    161. ^ "Mean monthly rainfall". National Institute of Water and Atmospheric Research. Archived from the original (XLS) on 3 May 2011. Retrieved 4 February 2011.
                    162. ^ "Mean monthly sunshine hours". National Institute of Water and Atmospheric Research. Archived from the original (XLS) on 15 October 2008. Retrieved 4 February 2011.
                    163. ^ "New Zealand climate and weather". Tourism New Zealand. Retrieved 13 November 2016.
                    164. ^ "Climate data and activities". National Institute of Water and Atmospheric Research. Retrieved 11 February 2016.
                    165. ^ Cooper, R.; Millener, P. (1993). "The New Zealand biota: Historical background and new research". Trends in Ecology & Evolution. 8 (12): 429. doi:10.1016/0169-5347(93)90004-9.
                    166. ^ Trewick SA, Morgan-Richards M. 2014. New Zealand Wild Life. Penguin, New Zealand. ISBN 9780143568896
                    167. ^ Lindsey, Terence; Morris, Rod (2000). Collins Field Guide to New Zealand Wildlife. HarperCollins (New Zealand) Limited. p. 14. ISBN 978-1-86950-300-0.
                    168. ^ a b "Frequently asked questions about New Zealand plants". New Zealand Plant Conservation Network. May 2010. Retrieved 15 January 2011.
                    169. ^ De Lange, Peter James; Sawyer, John William David & Rolfe, Jeremy (2006). New Zealand indigenous vascular plant checklist. New Zealand Plant Conservation Network. ISBN 0-473-11306-6.
                    170. ^ Wassilieff, Maggy (March 2009). "Lichens – Lichens in New Zealand". Te Ara: The Encyclopedia of New Zealand. Retrieved 16 January 2011.
                    171. ^ McLintock, Alexander, ed. (April 2010) [originally published in 1966]. Mixed Broadleaf Podocarp and Kauri Forest. An Encyclopaedia of New Zealand. Retrieved 15 January 2011.
                    172. ^ Mark, Alan (March 2009). "Grasslands – Tussock grasslands". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 January 2010.
                    173. ^ "Commentary on Forest Policy in the Asia-Pacific Region (A Review for Indonesia, Malaysia, New Zealand, Papua New Guinea, Philippines, Thailand and Western Samoa)". Forestry Department. 1997. Retrieved 4 February 2011.
                    174. ^ McGlone, M.S. (1989). "The Polynesian settlement of New Zealand in relation to environmental and biotic changes" (PDF). New Zealand Journal of Ecology. 12(S): 115–129. Archived from the original (PDF) on 17 July 2014.
                    175. ^ Taylor, R. and Smith, I. (1997). The state of New Zealand’s environment 1997. Ministry for the Environment, Wellington.
                    176. ^ "New Zealand ecology: Flightless birds". TerraNature. Retrieved 17 January 2011.
                    177. ^ a b Holdaway, Richard (March 2009). "Extinctions – New Zealand extinctions since human arrival". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    178. ^ Kirby, Alex (January 2005). "Huge eagles 'dominated NZ skies'". BBC News. Retrieved 4 February 2011.
                    179. ^ "Reptiles and frogs". Department of Conservation. Archived from the original on 29 January 2015. Retrieved 25 June 2017.
                    180. ^ Pollard, Simon (September 2007). "Spiders and other arachnids". Te Ara: The Encyclopedia of New Zealand. Retrieved 25 June 2017.
                    181. ^ "Wētā". Department of Conservation. Retrieved 25 June 2017.
                    182. ^ Ryan, Paddy (March 2009). "Snails and slugs – Flax snails, giant snails and veined slugs". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    183. ^ Herrera-Flores, Jorge A.; Stubbs, Thomas L.; Benton, Michael J.; Ruta, Marcello (May 2017). "Macroevolutionary patterns in Rhynchocephalia: is the tuatara (Sphenodon punctatus) a living fossil?". Palaeontology. 60 (3): 319–328. doi:10.1111/pala.12284.
                    184. ^ "Tiny Bones Rewrite Textbooks, first New Zealand land mammal fossil". University of New South Wales. 31 May 2007. Archived from the original on 31 May 2007.
                    185. ^ Worthy, Trevor H.; Tennyson, Alan J. D.; Archer, Michael; Musser, Anne M.; Hand, Suzanne J.; Jones, Craig; Douglas, Barry J.; McNamara, James A.; Beck, Robin M. D. (2006). "Miocene mammal reveals a Mesozoic ghost lineage on insular New Zealand, southwest Pacific". Proceedings of the National Academy of Sciences of the United States of America. 103 (51): 19419–23. Bibcode:2006PNAS..10319419W. doi:10.1073/pnas.0605684103. PMC 1697831. PMID 17159151.
                    186. ^ "Marine Mammals". Department of Conservation. Archived from the original on 8 March 2011. Retrieved 17 January 2011.
                    187. ^ "Sea and shore birds". Department of Conservation. Retrieved 7 March 2011.
                    188. ^ "Penguins". Department of Conservation. Retrieved 7 March 2011.
                    189. ^ Jones, Carl (2002). "Reptiles and Amphibians". In Perrow, Martin; Davy, Anthony (eds.). Handbook of ecological restoration: Principles of Restoration. 2. Cambridge University Press. p. 362. ISBN 0-521-79128-6.
                    190. ^ Towns, D.; Ballantine, W. (1993). "Conservation and restoration of New Zealand Island ecosystems". Trends in Ecology & Evolution. 8 (12): 452. doi:10.1016/0169-5347(93)90009-E.
                    191. ^ Rauzon, Mark (2008). "Island restoration: Exploring the past, anticipating the future" (PDF). Marine Ornithology. 35: 97–107.
                    192. ^ Diamond, Jared (1990). Towns, D; Daugherty, C; Atkinson, I (eds.). New Zealand as an archipelago: An international perspective (PDF). Wellington: Conservation Sciences Publication No. 2. Department of Conservation. pp. 3–8.
                    193. ^ World Economic Outlook. International Monetary Fund. April 2018. p. 63. ISBN 978-1-48434-971-7. Retrieved 21 June 2018.
                    194. ^ "Rankings on Economic Freedom". The Heritage Foundation. 2016. Retrieved 30 November 2016.
                    195. ^ "Currencies of the territories listed in the BS exchange rate lists". Bank of Slovenia. Retrieved 22 January 2011.
                    196. ^ a b c McLintock, Alexander, ed. (November 2009) [originally published in 1966]. Historical evolution and trade patterns. An Encyclopaedia of New Zealand. Retrieved 10 February 2011.
                    197. ^ Stringleman, Hugh; Peden, Robert (October 2009). "Sheep farming – Growth of the frozen meat trade, 1882–2001". Te Ara: The Encyclopedia of New Zealand. Retrieved 6 May 2010.
                    198. ^ Baker, John (February 2010) [1966]. McLintock, Alexander (ed.). Some Indicators of Comparative Living Standards. An Encyclopaedia of New Zealand. Retrieved 30 April 2010. PDF Table
                    199. ^ Wilson, John (March 2009). "History – The later 20th century". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 February 2011.
                    200. ^ Nixon, Chris; Yeabsley, John (April 2010). "Overseas trade policy – Difficult times – the 1970s and early 1980s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    201. ^ Evans, N. "Up From Down Under: After a Century of Socialism, Australia and New Zealand are Cutting Back Government and Freeing Their Economies". National Review. 46 (16): 47–51.
                    202. ^ Trade, Food Security, and Human Rights: The Rules for International Trade in Agricultural Products and the Evolving World Food Crisis. Routledge. 2016. p. 125. ISBN 9781317008521.
                    203. ^ Wayne Arnold (2 August 2007). "Surviving Without Subsidies". The New York Times. Retrieved 11 August 2015. ... ever since a liberal but free-market government swept to power in 1984 and essentially canceled handouts to farmers ... They went cold turkey and in the process it was very rough on their farming economy
                    204. ^ Easton, Brian (November 2010). "Economic history – Government and market liberalisation". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    205. ^ Hazledine, Tim (1998). Taking New Zealand Seriously: The Economics of Decency (PDF). HarperCollins Publishers. ISBN 1-86950-283-3. Archived from the original (PDF) on 10 May 2011.
                    206. ^ "NZ tops Travellers' Choice Awards". Stuff Travel. May 2008. Retrieved 30 April 2010.
                    207. ^ a b c "Unemployment: the Social Report 2016 – Te pūrongo oranga tangata". Ministry of Social Development. Retrieved 18 August 2017.
                    208. ^ "New Zealand Takes a Pause in Cutting Rates". The New York Times. 10 June 2009. Retrieved 30 April 2010.
                    209. ^ "New Zealand's slump longest ever". BBC News. 26 June 2009. Retrieved 30 April 2010.
                    210. ^ Bascand, Geoff (February 2011). "Household Labour Force Survey: December 2010 quarter – Media Release". Statistics New Zealand. Archived from the original on 29 April 2011. Retrieved 4 February 2011.
                    211. ^ Davenport, Sally (2004). "Panic and panacea: brain drain and science and technology human capital policy". Research Policy. 33 (4): 617–630. doi:10.1016/j.respol.2004.01.006.
                    212. ^ O'Hare, Sean (September 2010). "New Zealand brain-drain worst in world". The Daily Telegraph. United Kingdom.
                    213. ^ Collins, Simon (March 2005). "Quarter of NZ's brightest are gone". New Zealand Herald.
                    214. ^ Winkelmann, Rainer (2000). "The labour market performance of European immigrants in New Zealand in the 1980s and 1990s". The International Migration Review. The Center for Migration Studies of New York. 33 (1): 33–58. doi:10.2307/2676011. JSTOR 2676011. Journal subscription required
                    215. ^ Bain 2006, p. 44.
                    216. ^ "GII 2016 Report" (PDF). Global Innovation Index. Retrieved 21 June 2018.
                    217. ^ Groser, Tim (March 2009). "Speech to ASEAN-Australia-New Zealand Free Trade Agreement Seminars". New Zealand Government. Retrieved 30 January 2011.
                    218. ^ "Improving Access to Markets:Agriculture". New Zealand Ministry of Foreign Affairs and Trade. Retrieved 22 January 2011.
                    219. ^ "Standard International Trade Classification R4 – Exports (Annual-Jun)". Statistics New Zealand. April 2015. Retrieved 3 April 2015.
                    220. ^ a b c "Goods and services trade by country: Year ended June 2018 – corrected". Statistics New Zealand. Retrieved 17 February 2019.
                    221. ^ "China and New Zealand sign free trade deal". The New York Times. April 2008.
                    222. ^ a b "Key Tourism Statistics" (PDF). Ministry of Business, Innovation and Employment. 26 April 2017. Retrieved 26 April 2017.
                    223. ^ Easton, Brian (March 2009). "Economy – Agricultural production". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    224. ^ Stringleman, Hugh; Peden, Robert (March 2009). "Sheep farming – Changes from the 20th century". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    225. ^ Stringleman, Hugh; Scrimgeour, Frank (November 2009). "Dairying and dairy products – Dairying in the 2000s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    226. ^ Stringleman, Hugh; Scrimgeour, Frank (March 2009). "Dairying and dairy products – Dairy exports". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    227. ^ Stringleman, Hugh; Scrimgeour, Frank (March 2009). "Dairying and dairy products – Manufacturing and marketing in the 2000s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    228. ^ Dalley, Bronwyn (March 2009). "Wine – The wine boom, 1980s and beyond". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    229. ^ "Wine in New Zealand". The Economist. 27 March 2008. Retrieved 29 April 2017.
                    230. ^ "Agricultural and forestry exports from New Zealand: Primary sector export values for the year ending June 2010". Ministry of Agriculture and Forestry. 14 January 2011. Archived from the original on 10 May 2011. Retrieved 8 April 2011.
                    231. ^ a b Energy in New Zealand 2016 (PDF) (Report). Ministry of Business, Innovation and Employment. September 2016. p. 47. ISSN 2324-5913. Archived from the original (PDF) on 3 May 2017.
                    232. ^ "Appendix 1: Technical information about drinking water supply in the eight local authorities". Office of the Auditor-General. Retrieved 2 September 2016.
                    233. ^ "Water supply". Greater Wellington Regional Council. Retrieved 2 September 2016.
                    234. ^ "State highway frequently asked questions". NZ Transport Agency. Retrieved 28 April 2017.
                    235. ^ Humphris, Adrian (April 2010). "Public transport – Passenger trends". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    236. ^ Atkinson, Neill (November 2010). "Railways – Rail transformed". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    237. ^ "About Metlink". Retrieved 27 December 2016.
                    238. ^ Atkinson, Neill (April 2010). "Railways – Freight transport". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    239. ^ "International Visitors" (PDF). Ministry of Economic Development. June 2009. Retrieved 30 January 2011.
                    240. ^ "10. Airports". Infrastructure Stocktake: Infrastructure Audit. Ministry of Economic Development. December 2005. Archived from the original on 22 May 2010. Retrieved 30 January 2011.
                    241. ^ a b Wilson, A. C. (March 2010). "Telecommunications - Telecom". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 August 2017.
                    242. ^ "Telecom separation". Ministry of Business, Innovation and Employment. 14 September 2015. Retrieved 11 August 2017.
                    243. ^ "Broadband and mobile programmes - Ministry of Business, Innovation & Employment". www.mbie.govt.nz.
                    244. ^ "2017 Global ICT Development Index". International Telecommunication Union (ITU). 2018. Retrieved 18 September 2018.
                    245. ^ "2013 Census usually resident population counts". Statistics New Zealand. 14 October 2013. Retrieved 20 September 2018.
                    246. ^ "National population projections: 2016(base)–2068" (Press release). Statistics New Zealand. 18 October 2016. Retrieved 20 October 2018.
                    247. ^ "Subnational population estimates at 30 June 2009". Statistics New Zealand. 30 June 2007. Retrieved 30 April 2010.
                    248. ^ "Quality of Living Ranking 2016". London: Mercer. 23 February 2016. Retrieved 28 April 2017.
                    249. ^ "NZ life expectancy among world's best". Stuff.co.nz. Fairfax NZ. Retrieved 6 July 2014.
                    250. ^ a b Department of Economic and Social Affairs Population Division (2009). "World Population Prospects" (PDF). 2008 revision. United Nations. Retrieved 29 August 2009.
                    251. ^ "World Factbook EUROPE : NEW ZEALAND", The World Factbook, 12 July 2018
                    252. ^ "New Zealand mortality statistics: 1950 to 2010" (PDF). Ministry of Health of New Zealand. 2 March 2011. Retrieved 16 November 2016.
                    253. ^ "Health expenditure and financing". stats.oecd.org. OECD. 2016. Retrieved 8 December 2017.
                    254. ^ "Subnational Population Estimates: At 30 June 2018 (provisional)". Statistics New Zealand. 23 October 2018. Retrieved 23 October 2018. For urban areas, "Subnational population estimates (UA, AU), by age and sex, at 30 June 1996, 2001, 2006-18 (2017 boundaries)". Statistics New Zealand. 23 October 2018. Retrieved 23 October 2018.
                    255. ^ "2013 Census QuickStats about culture and identity – Ethnic groups in New Zealand". Statistics New Zealand. Retrieved 29 August 2014.
                    256. ^ Pool, Ian (May 2011). "Population change - Key population trends". Te Ara: The Encyclopedia of New Zealand. Archived from the original on 18 August 2017. Retrieved 18 August 2017.
                    257. ^ Dalby, Simon (September 1993). "The 'Kiwi disease': geopolitical discourse in Aotearoa/New Zealand and the South Pacific". Political Geography. 12 (5): 437–456. doi:10.1016/0962-6298(93)90012-V.
                    258. ^ Callister, Paul (2004). "Seeking an Ethnic Identity: Is "New Zealander" a Valid Ethnic Category?" (PDF). New Zealand Population Review. 30 (1&2): 5–22.
                    259. ^ Bain 2006, p. 31.
                    260. ^ "Draft Report of a Review of the Official Ethnicity Statistical Standard: Proposals to Address the 'New Zealander' Response Issue" (PDF). Statistics New Zealand. April 2009. Retrieved 18 January 2011.
                    261. ^ Ranford, Jodie. "'Pakeha', Its Origin and Meaning". Māori News. Retrieved 20 February 2008. Originally the Pakeha were the early European settlers, however, today ‘Pakeha’ is used to describe any peoples of non-Maori or non-Polynesian heritage. Pakeha is not an ethnicity but rather a way to differentiate between the historical origins of our settlers, the Polynesians and the Europeans, the Maori and the other
                    262. ^ Socidad Peruana de Medicina Intensiva (SOPEMI) (2000). Trends in international migration: continuous reporting system on migration. Organisation for Economic Co-operation and Development. pp. 276–278.
                    263. ^ Walrond, Carl (21 September 2007). "Dalmatians". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 April 2010.
                    264. ^ "Peoples". Te Ara: The Encyclopedia of New Zealand. 2005. Retrieved 2 June 2017.
                    265. ^ a b Phillips, Jock (11 August 2015). "History of immigration". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 June 2017.
                    266. ^ Brawley, Sean (1993). "'No White Policy in NZ': Fact and Fiction in New Zealand's Asian Immigration Record, 1946-1978" (PDF). New Zealand Journal of History. 27 (1): 33–36. Retrieved 2 June 2017.
                    267. ^ "International Migration Outlook – New Zealand 2009/10" (PDF). New Zealand Department of Labour. 2010. p. 2. ISSN 1179-5085. Archived from the original (PDF) on 11 May 2011. Retrieved 16 April 2011.
                    268. ^ "2013 Census QuickStats about culture and identity – Birthplace and people born overseas". Statistics New Zealand. Retrieved 29 August 2014.
                    269. ^ Butcher, Andrew; McGrath, Terry (2004). "International Students in New Zealand: Needs and Responses" (PDF). International Education Journal. 5 (4).
                    270. ^ 2013 Census QuickStats, Statistics New Zealand, 2013, ISBN 978-0-478-40864-5
                    271. ^ a b c "2013 Census QuickStats about culture and identity – Languages spoken". Statistics New Zealand. Retrieved 8 September 2016.
                    272. ^ Hay, Maclagan & Gordon 2008, p. 14.
                    273. ^ * Bauer, Laurie; Warren, Paul; Bardsley, Dianne; Kennedy, Marianna; Major, George (2007), "New Zealand English", Journal of the International Phonetic Association, 37 (1): 97–102, doi:10.1017/S0025100306002830
                    274. ^ a b Phillips, Jock (March 2009). "The New Zealanders – Bicultural New Zealand". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    275. ^ Squires, Nick (May 2005). "British influence ebbs as New Zealand takes to talking Māori". The Daily Telegraph. Retrieved 3 May 2017.
                    276. ^ "Waitangi Tribunal claim – Māori Language Week". Ministry for Culture and Heritage. July 2010. Retrieved 19 January 2011.
                    277. ^ "Ngā puna kōrero: Where Māori speak te reo – infographic". Statistics New Zealand. Retrieved 8 September 2016.
                    278. ^ Drinnan, John (8 July 2016). "'Maori' will remain in the name Maori Television". New Zealand Herald. Retrieved 28 August 2016. According to 2015 figures supplied by Maori TV, its two channels broadcast an average of 72 per cent Maori language content - 59 per cent on the main channel and 99 per cent on te reo.
                    279. ^ "Ngāi Tahu Claims Settlement Act 1998". New Zealand Parliamentary Counsel Office. 20 May 2014 [1 October 1998]. Retrieved 10 March 2019.
                    280. ^ New Zealand Sign Language Act 2006 No 18 (as at 30 June 2008), Public Act. New Zealand Parliamentary Counsel Office. Retrieved 29 November 2011.
                    281. ^ Zuckerman, Phil (2006). Martin, Michael (ed.). The Cambridge Companion to Atheism (PDF). Cambridge University Press. pp. 47–66. ISBN 978-0-521-84270-9. Retrieved 8 August 2017.
                    282. ^ Walrond, Carl (May 2012). "Atheism and secularism – Who is secular?". Te Ara: The Encyclopedia of New Zealand. Retrieved 8 August 2017.
                    283. ^ a b c "2013 Census QuickStats about culture and identity – tables". Statistics New Zealand. 15 April 2014. Retrieved 14 April 2018. Excel download
                    284. ^ Kaa, Hirini (May 2011). "Māori and Christian denominations". Te Ara: The Encyclopedia of New Zealand. Retrieved 20 April 2017.
                    285. ^ Morris, Paul (May 2011). "Diverse religions". Te Ara: The Encyclopedia of New Zealand. Retrieved 20 April 2017.
                    286. ^ a b Dench, Olivia (July 2010). "Education Statistics of New Zealand: 2009". Education Counts. Retrieved 19 January 2011.
                    287. ^ "Education Act 1989 No 80". New Zealand Parliamentary Counsel Office. 1989. Section 3. Retrieved 5 January 2013.
                    288. ^ "Education Act 1989 No 80 (as at 01 February 2011), Public Act. Part 14: Establishment and disestablishment of tertiary institutions, Section 62: Establishment of institutions". Education Act 1989 No 80. New Zealand Parliamentary Counsel Office. 1 February 2011. Retrieved 15 August 2011.
                    289. ^ "Studying in New Zealand: Tertiary education". New Zealand Qualifications Authority. Retrieved 15 August 2011.
                    290. ^ "Educational attainment of the population". Education Counts. 2006. Archived from the original (xls) on 15 October 2008. Retrieved 21 February 2008.
                    291. ^ "What Students Know and Can Do: Student Performance in Reading, Mathematics and Science 2010" (PDF). OECD. Retrieved 21 July 2012.
                    292. ^ Kennedy 2007, p. 398.
                    293. ^ Hearn, Terry (March 2009). "English – Importance and influence". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    294. ^ "Conclusions – British and Irish immigration". Ministry for Culture and Heritage. March 2007. Retrieved 21 January 2011.
                    295. ^ Stenhouse, John (November 2010). "Religion and society – Māori religion". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    296. ^ "Māori Social Structures". Ministry of Justice. March 2001. Retrieved 21 January 2011.
                    297. ^ "Thousands turn out for Pasifika Festival". Radio New Zealand. 25 March 2017. Retrieved 18 August 2017.
                    298. ^ Kennedy 2007, p. 400.
                    299. ^ Kennedy 2007, p. 399.
                    300. ^ Phillips, Jock (March 2009). "The New Zealanders – Post-war New Zealanders". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    301. ^ Phillips, Jock (March 2009). "The New Zealanders – Ordinary blokes and extraordinary sheilas". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    302. ^ Phillips, Jock (March 2009). "Rural mythologies – The cult of the pioneer". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    303. ^ Barker, Fiona (June 2012). "New Zealand identity – Culture and arts". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 December 2016.
                    304. ^ a b Wilson, John (September 2016). "Nation and government – Nationhood and identity". Te Ara: The Encyclopedia of New Zealand. Retrieved 3 December 2016.
                    305. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Visual arts and crafts". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    306. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Elements of Carving. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    307. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Surface Patterns. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    308. ^ McKay, Bill (2004). "Māori architecture: transforming western notions of architecture". Fabrications. 14 (1&2): 1–12. doi:10.1080/10331867.2004.10525189. Archived from the original on 13 May 2011.
                    309. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Painted Designs. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    310. ^ McLintock, Alexander, ed. (2009) [1966]. Tattooing. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    311. ^ a b "Beginnings – history of NZ painting". Ministry for Culture and Heritage. December 2010. Retrieved 17 February 2011.
                    312. ^ "A new New Zealand art – history of NZ painting". Ministry for Culture and Heritage. November 2010. Retrieved 16 February 2011.
                    313. ^ "Contemporary Maori art". Ministry for Culture and Heritage. November 2010. Retrieved 16 February 2011.
                    314. ^ Rauer, Julie. "Paradise Lost: Contemporary Pacific Art At The Asia Society". Asia Society and Museum. Retrieved 17 February 2011.
                    315. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Textile Designs. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    316. ^ Keane, Basil (March 2009). "Pounamu – jade or greenstone – Implements and adornment". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 February 2011.
                    317. ^ Wilson, John (March 2009). "Society – Food, drink and dress". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 February 2011.
                    318. ^ Swarbrick, Nancy (June 2010). "Creative life – Design and fashion". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    319. ^ a b "Fashion in New Zealand – New Zealand's fashion industry". The Economist. 28 February 2008. Retrieved 6 August 2009.
                    320. ^ Swarbrick, Nancy (June 2010). "Creative life – Writing and publishing". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    321. ^ "The making of New Zealand literature". Ministry for Culture and Heritage. November 2010. Retrieved 22 January 2011.
                    322. ^ "New directions in the 1930s – New Zealand literature". Ministry for Culture and Heritage. August 2008. Retrieved 12 February 2011.
                    323. ^ "The war and beyond – New Zealand literature". Ministry for Culture and Heritage. November 2007. Retrieved 12 February 2011.
                    324. ^ "28 cities join the UNESCO Creative Cities Network". UNESCO. December 2014. Retrieved 7 March 2015.
                    325. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Music". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    326. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Maori Music. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    327. ^ McLintock, Alexander, ed. (April 2009) [1966]. Musical Instruments. An Encyclopaedia of New Zealand. Retrieved 16 February 2011.
                    328. ^ McLintock, Alexander, ed. (April 2009) [1966]. Instruments Used for Non-musical Purposes. An Encyclopaedia of New Zealand. Retrieved 16 February 2011.
                    329. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: General History. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    330. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: Brass Bands. An Encyclopaedia of New Zealand. Retrieved 14 April 2011.
                    331. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: Pipe Bands. An Encyclopaedia of New Zealand. Retrieved 14 April 2011.
                    332. ^ Swarbrick, Nancy (June 2010). "Creative life – Performing arts". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    333. ^ "History – celebrating our music since 1965". Recording Industry Association of New Zealand. 2008. Archived from the original on 14 September 2011. Retrieved 23 January 2012.
                    334. ^ "About RIANZ – Introduction". Recording Industry Association of New Zealand. Archived from the original on 21 December 2011. Retrieved 23 January 2012.
                    335. ^ Downes, Siobhan (1 January 2017). "World famous in New Zealand: Hobbiton Movie Set". Stuff Travel. Retrieved 6 July 2017.
                    336. ^ Brian, Pauling (October 2014). "Radio – The early years, 1921 to 1932". Te Ara: The Encyclopedia of New Zealand. Retrieved 6 July 2017.
                    337. ^ "New Zealand's first official TV broadcast". Ministry for Culture and Heritage. December 2016. Retrieved 6 July 2017.
                    338. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Film and broadcasting". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    339. ^ Horrocks, Roger. "A History of Television in New Zealand". NZ On Screen. Retrieved 13 September 2017.
                    340. ^ "Top 10 Highest Grossing New Zealand Movies Ever". Flicks.co.nz. May 2016. Retrieved 11 August 2017.
                    341. ^ Cieply, Michael; Rose, Jeremy (October 2010). "New Zealand Bends and 'Hobbit' Stays". The New York Times. Retrieved 11 August 2017.
                    342. ^ "Production Guide: Locations". Film New Zealand. Archived from the original on 7 November 2010. Retrieved 21 January 2011.
                    343. ^ Myllylahti, Merja (December 2016). JMAD New Zealand Media Ownership Report 2016 (PDF) (Report). Auckland University of Technology. pp. 4–29. Archived from the original (PDF) on 21 May 2017. Retrieved 11 August 2017.
                    344. ^ "Scores and Status Data 1980-2015". Freedom of the Press 2015. Freedom House. Retrieved 23 November 2016.
                    345. ^ Hearn, Terry (March 2009). "English – Popular culture". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2012.
                    346. ^ "Sport, Fitness and Leisure". New Zealand Official Yearbook. Statistics New Zealand. 2000. Archived from the original on 7 June 2011. Retrieved 21 July 2008. Traditionally New Zealanders have excelled in rugby union, which is regarded as the national sport, and track and field athletics.
                    347. ^ a b Phillips, Jock (February 2011). "Sports and leisure – Organised sports". Te Ara: The Encyclopedia of New Zealand. Retrieved 23 March 2011.
                    348. ^ a b "More and more students wear school sports colours". New Zealand Secondary School Sports Council. Retrieved 30 March 2015.
                    349. ^ Crawford, Scott (January 1999). "Rugby and the Forging of National Identity". In Nauright, John (ed.). Sport, Power And Society In New Zealand: Historical And Contemporary Perspectives (PDF). ASSH Studies In Sports History. Archived from the original (PDF) on 19 January 2012. Retrieved 22 January 2011.
                    350. ^ "Rugby, racing and beer". Ministry for Culture and Heritage. August 2010. Retrieved 22 January 2011.
                    351. ^ Derby, Mark (December 2010). "Māori–Pākehā relations – Sports and race". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    352. ^ Bain 2006, p. 69.
                    353. ^ Langton, Graham (1996). A history of mountain climbing in New Zealand to 1953 (Thesis). Christchurch: University of Canterbury. p. 28. Retrieved 12 August 2017.
                    354. ^ "World mourns Sir Edmund Hillary". The Age. Melbourne. 11 January 2008.
                    355. ^ "Sport and Recreation Participation Levels" (PDF). Sport and Recreation New Zealand. 2009. Archived from the original (PDF) on 15 January 2015. Retrieved 27 November 2016.
                    356. ^ Barclay-Kerr, Hoturoa (September 2013). "Waka ama – outrigger canoeing". Te Ara: The Encyclopedia of New Zealand. Retrieved 12 August 2017.
                    357. ^ "NZ's first Olympic century". Ministry for Culture and Heritage. August 2016. Retrieved 27 April 2017.
                    358. ^ "London 2012 Olympic Games: Medal strike rate – Final count (revised)". Statistics New Zealand. 14 August 2012. Retrieved 4 December 2013.
                    359. ^ "Rio 2016 Olympic Games: Medals per capita". Statistics New Zealand. 30 August 2016. Retrieved 27 April 2017.
                    360. ^ Kerr, James (14 November 2013). "The All Blacks guide to being successful (off the field)". The Daily Telegraph. London. Retrieved 4 December 2013.
                    361. ^ Fordyce, Tom (23 October 2011). "2011 Rugby World Cup final: New Zealand 8-7 France". BBC Sport. Retrieved 4 December 2013.
                    362. ^ a b "New Zealand Cuisine". New Zealand Tourism Guide. January 2016. Retrieved 4 January 2016.
                    363. ^ Petrie, Hazel (November 2008). "Kai Pākehā – introduced foods". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 June 2017.
                    364. ^ Whaanga, Mere (June 2006). "Mātaitai – shellfish gathering". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 June 2017.
                    365. ^ "Story: Shellfish". Te Ara: The Encyclopedia of New Zealand. Retrieved 29 August 2016.
                    366. ^ Burton, David (September 2013). "Cooking – Cooking methods". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 December 2016.
                    367. ^ Royal, Charles; Kaka-Scott, Jenny (September 2013). "Māori foods – kai Māori". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 September 2016.

                    References

                    • Alley, Roderic (2008). New Zealand in World Affairs IV 1990–2005. New Zealand: Victoria University Press. ISBN 978-0-864-73548-5.
                    • Bain, Carolyn (2006). New Zealand. Lonely Planet. ISBN 1-74104-535-5.
                    • Garden, Donald (2005). Stoll, Mark (ed.). Australia, New Zealand, and the Pacific: An Environmental History. Nature and Human Societies. ABC-CLIO/Greenwood. ISBN 978-1-57607-868-6.
                    • Hay, Jennifer; Maclagan, Margaret; Gordon, Elizabeth (2008). Dialects of English: New Zealand English. Edinburgh University Press. ISBN 978-0-7486-2529-1.
                    • Kennedy, Jeffrey (2007). "Leadership and Culture in New Zealand". In Chhokar, Jagdeep; Brodbeck, Felix; House, Robert (eds.). Culture and Leadership Across the World: The GLOBE Book of In-Depth Studies of 25 Societies. United States: Psychology Press. ISBN 978-0-8058-5997-3.
                    • King, Michael (2003). The Penguin History of New Zealand. New Zealand: Penguin Books. ISBN 978-0-14-301867-4.
                    • Mein Smith, Philippa (2005). A Concise History of New Zealand. Australia: Cambridge University Press. ISBN 0-521-54228-6.
                    • Smelt, Roselynn; Jui Lin, Yong (2009). New Zealand. Cultures of the World (2nd ed.). New York: Marshall Cavendish. ISBN 978-0-7614-3415-3.

                    Further reading

                    External links

                    Government
                    Travel
                    General Information
                    ================================================ FILE: test/test-pages/wikipedia-2/source.html ================================================ New Zealand - Wikipedia
                    This is a good article. Follow the link for more information.
                    Page semi-protected

                    New Zealand

                    From Wikipedia, the free encyclopedia
                    Jump to navigation Jump to search
                    This article is about the country. For other uses, see New Zealand (disambiguation).
                    "NZ" redirects here. For other uses, see NZ (disambiguation).

                    Coordinates: 42°S 174°E / 42°S 174°E / -42; 174

                    New Zealand

                    Aotearoa  (Māori)
                    A map of the hemisphere centred on New Zealand, using an orthographic projection.
                    Location of New Zealand, including outlying islands, its territorial claim in the Antarctic, and Tokelau
                    Capital Wellington
                    41°17′S 174°27′E / 41.283°S 174.450°E / -41.283; 174.450
                    Largest city Auckland
                    Official languages
                    Ethnic groups
                    (2018)
                    Demonym(s) New Zealander
                    Kiwi (informal)
                    Government Unitary parliamentary constitutional monarchy
                    • Monarch
                    		Elizabeth II
                    Patsy Reddy
                    Jacinda Ardern
                    Legislature Parliament
                    (House of Representatives)
                    Stages of independence 
                    from the United Kingdom
                    7 May 1856
                    • Dominion
                    		26 September 1907

                    25 November 1947
                    Area
                    • Total
                    		268,021 km2 (103,483 sq mi) (75th)
                    • Water (%)
                    		1.6[n 4]
                    Population
                    • September 2019 estimate
                    		4,933,210[5] (120th)
                    • 2018 census
                    		4,699,755
                    • Density
                    		18.2/km2 (47.1/sq mi) (203rd)
                    GDP (PPP) 2018 estimate
                    • Total
                    		$199 billion[6]
                    • Per capita
                    		$40,266[6]
                    GDP (nominal) 2018 estimate
                    • Total
                    		$206 billion[6]
                    • Per capita
                    		$41,616[6]
                    Gini (2014) 33.0[7]
                    medium · 22nd
                    HDI (2017) Increase 0.917[8]
                    very high · 16th
                    Currency New Zealand dollar ($) (NZD)
                    Time zone UTC+12 (NZST[n 5])
                    • Summer (DST)
                    		UTC+13 (NZDT[n 6])
                    Date format dd/mm/yyyy
                    yyyy-mm-dd[10]
                    Driving side left
                    Calling code +64
                    ISO 3166 code NZ
                    Internet TLD .nz

                    New Zealand (Māori: Aotearoa [aɔˈtɛaɾɔa]) is a sovereign island country in the southwestern Pacific Ocean. The country geographically comprises two main landmasses—the North Island (Te Ika-a-Māui), and the South Island (Te Waipounamu)—and around 600 smaller islands. It has a total land area of 268,000 square kilometres (103,500 sq mi). New Zealand is situated some 2,000 kilometres (1,200 mi) east of Australia across the Tasman Sea and roughly 1,000 kilometres (600 mi) south of the Pacific island areas of New Caledonia, Fiji, and Tonga. Because of its remoteness, it was one of the last lands to be settled by humans. During its long period of isolation, New Zealand developed a distinct biodiversity of animal, fungal, and plant life. The country's varied topography and its sharp mountain peaks, such as the Southern Alps, owe much to the tectonic uplift of land and volcanic eruptions. New Zealand's capital city is Wellington, while its most populous city is Auckland.

                    Sometime between 1250 and 1300, Polynesians settled in the islands that later were named New Zealand and developed a distinctive Māori culture. In 1642, Dutch explorer Abel Tasman became the first European to sight New Zealand. In 1840, representatives of the United Kingdom and Māori chiefs signed the Treaty of Waitangi, which declared British sovereignty over the islands. In 1841, New Zealand became a colony within the British Empire and in 1907 it became a dominion; it gained full statutory independence in 1947 and the British monarch remained the head of state. Today, the majority of New Zealand's population of 4.9 million is of European descent; the indigenous Māori are the largest minority, followed by Asians and Pacific Islanders. Reflecting this, New Zealand's culture is mainly derived from Māori and early British settlers, with recent broadening arising from increased immigration. The official languages are English, Māori, and New Zealand Sign Language, with English being very dominant.

                    A developed country, New Zealand ranks highly in international comparisons of national performance, such as quality of life, health, education, protection of civil liberties, and economic freedom. New Zealand underwent major economic changes during the 1980s, which transformed it from a protectionist to a liberalised free-trade economy. The service sector dominates the national economy, followed by the industrial sector, and agriculture; international tourism is a significant source of revenue. Nationally, legislative authority is vested in an elected, unicameral Parliament, while executive political power is exercised by the Cabinet, led by the prime minister, currently Jacinda Ardern. Queen Elizabeth II is the country's monarch and is represented by a governor-general, currently Dame Patsy Reddy. In addition, New Zealand is organised into 11 regional councils and 67 territorial authorities for local government purposes. The Realm of New Zealand also includes Tokelau (a dependent territory); the Cook Islands and Niue (self-governing states in free association with New Zealand); and the Ross Dependency, which is New Zealand's territorial claim in Antarctica. New Zealand is a member of the United Nations, Commonwealth of Nations, ANZUS, Organisation for Economic Co-operation and Development, ASEAN Plus Six, Asia-Pacific Economic Cooperation, the Pacific Community and the Pacific Islands Forum.

                    Etymology

                    See also: New Zealand place names
                    Brown square paper with Dutch writing and a thick red, curved line
                    Detail from a 1657 map showing the western coastline of "Nova Zeelandia". (In this map, north is at the bottom.)

                    Dutch explorer Abel Tasman sighted New Zealand in 1642 and named it Staten Land "in honour of the States General" (Dutch parliament). He wrote, "it is possible that this land joins to the Staten Land but it is uncertain",[11] referring to a landmass of the same name at the southern tip of South America, discovered by Jacob Le Maire in 1616.[12][13] In 1645, Dutch cartographers renamed the land Nova Zeelandia after the Dutch province of Zeeland.[14][15] British explorer James Cook subsequently anglicised the name to New Zealand.[16]

                    Aotearoa (pronounced /ˌaʊtɛəˈroʊ.ə/; often translated as "land of the long white cloud")[17] is the current Māori name for New Zealand. It is unknown whether Māori had a name for the whole country before the arrival of Europeans, with Aotearoa originally referring to just the North Island.[18] Māori had several traditional names for the two main islands, including Te Ika-a-Māui (the fish of Māui) for the North Island and Te Waipounamu (the waters of greenstone) or Te Waka o Aoraki (the canoe of Aoraki) for the South Island.[19] Early European maps labelled the islands North (North Island), Middle (South Island) and South (Stewart Island / Rakiura).[20] In 1830, mapmakers began to use "North" and "South" on their maps to distinguish the two largest islands and by 1907 this was the accepted norm.[16] The New Zealand Geographic Board discovered in 2009 that the names of the North Island and South Island had never been formalised, and names and alternative names were formalised in 2013. This set the names as North Island or Te Ika-a-Māui, and South Island or Te Waipounamu.[21] For each island, either its English or Māori name can be used, or both can be used together.[21]

                    History

                    Main article: History of New Zealand
                    One set of arrows point from Taiwan to Melanesia to Fiji/Samoa and then to the Marquesas Islands. The population then spread, some going south to New Zealand and others going north to Hawai'i. A second set start in southern Asia and end in Melanesia.
                    The Māori people are most likely descended from people who emigrated from Taiwan to Melanesia and then travelled east through to the Society Islands. After a pause of 70 to 265 years, a new wave of exploration led to the discovery and settlement of New Zealand.[22]

                    New Zealand was one of the last major landmasses settled by humans. Radiocarbon dating, evidence of deforestation[23] and mitochondrial DNA variability within Māori populations[24] suggest New Zealand was first settled by Eastern Polynesians between 1250 and 1300,[19][25] concluding a long series of voyages through the southern Pacific islands.[26] Over the centuries that followed, these settlers developed a distinct culture now known as Māori. The population was divided into iwi (tribes) and hapū (subtribes) who would sometimes cooperate, sometimes compete and sometimes fight against each other.[27] At some point a group of Māori migrated to Rēkohu, now known as the Chatham Islands, where they developed their distinct Moriori culture.[28][29] The Moriori population was all but wiped out between 1835 and 1862, largely because of Taranaki Māori invasion and enslavement in the 1830s, although European diseases also contributed. In 1862 only 101 survived, and the last known full-blooded Moriori died in 1933.[30]

                    An engraving of a sketched coastline on white background
                    Map of the New Zealand coastline as Cook charted it on his first visit in 1769–70. The track of the Endeavour is also shown.

                    The first Europeans known to have reached New Zealand were Dutch explorer Abel Tasman and his crew in 1642.[31] In a hostile encounter, four crew members were killed and at least one Māori was hit by canister shot.[32] Europeans did not revisit New Zealand until 1769 when British explorer James Cook mapped almost the entire coastline.[31] Following Cook, New Zealand was visited by numerous European and North American whaling, sealing and trading ships. They traded European food, metal tools, weapons and other goods for timber, Māori food, artefacts and water.[33] The introduction of the potato and the musket transformed Māori agriculture and warfare. Potatoes provided a reliable food surplus, which enabled longer and more sustained military campaigns.[34] The resulting intertribal Musket Wars encompassed over 600 battles between 1801 and 1840, killing 30,000–40,000 Māori.[35] From the early 19th century, Christian missionaries began to settle New Zealand, eventually converting most of the Māori population.[36] The Māori population declined to around 40% of its pre-contact level during the 19th century; introduced diseases were the major factor.[37]

                    A torn sheet of paper
                    The Waitangi sheet from the Treaty of Waitangi

                    In 1788 Captain Arthur Phillip assumed the position of Governor of the new British colony of New South Wales which according to his commission included New Zealand.[38] The British Government appointed James Busby as British Resident to New Zealand in 1832 following a petition from northern Māori.[39] In 1835, following an announcement of impending French settlement by Charles de Thierry, the nebulous United Tribes of New Zealand sent a Declaration of Independence to King William IV of the United Kingdom asking for protection.[39] Ongoing unrest, the proposed settlement of New Zealand by the New Zealand Company (which had already sent its first ship of surveyors to buy land from Māori) and the dubious legal standing of the Declaration of Independence prompted the Colonial Office to send Captain William Hobson to claim sovereignty for the United Kingdom and negotiate a treaty with the Māori.[40] The Treaty of Waitangi was first signed in the Bay of Islands on 6 February 1840.[41] In response to the New Zealand Company's attempts to establish an independent settlement in Wellington[42] and French settlers purchasing land in Akaroa,[43] Hobson declared British sovereignty over all of New Zealand on 21 May 1840, even though copies of the Treaty were still circulating throughout the country for Māori to sign.[44] With the signing of the Treaty and declaration of sovereignty the number of immigrants, particularly from the United Kingdom, began to increase.[45]

                    Black and white engraving depicting a crowd of people
                    A meeting of European and Māori inhabitants of Hawke's Bay Province. Engraving, 1863.

                    New Zealand, still part of the colony of New South Wales, became a separate Colony of New Zealand on 1 July 1841.[46] Armed conflict began between the Colonial government and Māori in 1843 with the Wairau Affray over land and disagreements over sovereignty. These conflicts, mainly in the North Island, saw thousands of Imperial troops and the Royal Navy come to New Zealand and became known as the New Zealand Wars. Following these armed conflicts, large amounts of Māori land was confiscated by the government to meet settler demands.[47]

                    The colony gained a representative government in 1852 and the first Parliament met in 1854.[48] In 1856 the colony effectively became self-governing, gaining responsibility over all domestic matters other than native policy.[48] (Control over native policy was granted in the mid-1860s.[48]) Following concerns that the South Island might form a separate colony, premier Alfred Domett moved a resolution to transfer the capital from Auckland to a locality near Cook Strait.[49] Wellington was chosen for its central location, with Parliament officially sitting there for the first time in 1865.[50]

                    In 1891 the Liberal Party came to power as the first organised political party.[51] The Liberal Government, led by Richard Seddon for most of its period in office,[52] passed many important social and economic measures. In 1893 New Zealand was the first nation in the world to grant all women the right to vote[51] and in 1894 pioneered the adoption of compulsory arbitration between employers and unions.[53]

                    In 1907, at the request of the New Zealand Parliament, King Edward VII proclaimed New Zealand a Dominion within the British Empire,[54] reflecting its self-governing status.[55] In 1947 the country adopted the Statute of Westminster, confirming that the British Parliament could no longer legislate for New Zealand without the consent of New Zealand.[48]

                    Early in the 20th century, New Zealand was involved in world affairs, fighting in the First and Second World Wars[56] and suffering through the Great Depression.[57] The depression led to the election of the First Labour Government and the establishment of a comprehensive welfare state and a protectionist economy.[58] New Zealand experienced increasing prosperity following the Second World War[59] and Māori began to leave their traditional rural life and move to the cities in search of work.[60] A Māori protest movement developed, which criticised Eurocentrism and worked for greater recognition of Māori culture and of the Treaty of Waitangi.[61] In 1975, a Waitangi Tribunal was set up to investigate alleged breaches of the Treaty, and it was enabled to investigate historic grievances in 1985.[41] The government has negotiated settlements of these grievances with many iwi,[62] although Māori claims to the foreshore and seabed have proved controversial in the 2000s.[63][64]

                    Government and politics

                    Main articles: Government of New Zealand and Politics of New Zealand
                    The Queen wearing her New Zealand insignia
                    Elizabeth II, Queen of New Zealand
                    A smiling woman wearing a black dress
                    Jacinda Ardern, Prime Minister of New Zealand

                    New Zealand is a constitutional monarchy with a parliamentary democracy,[65] although its constitution is not codified.[66] Elizabeth II is the Queen of New Zealand[67] and thus the head of state.[68] The Queen is represented by the governor-general, whom she appoints on the advice of the prime minister.[69] The governor-general can exercise the Crown's prerogative powers, such as reviewing cases of injustice and making appointments of ministers, ambassadors and other key public officials,[70] and in rare situations, the reserve powers (e.g. the power to dissolve parliament or refuse the royal assent of a bill into law).[71] The powers of the monarch and the governor-general are limited by constitutional constraints and they cannot normally be exercised without the advice of ministers.[71]

                    The New Zealand Parliament holds legislative power and consists of the Queen and the House of Representatives.[72] It also included an upper house, the Legislative Council, until this was abolished in 1950.[72] The supremacy of parliament over the Crown and other government institutions was established in England by the Bill of Rights 1689 and has been ratified as law in New Zealand.[72] The House of Representatives is democratically elected and a government is formed from the party or coalition with the majority of seats. If no majority is formed, a minority government can be formed if support from other parties during confidence and supply votes is assured.[72] The governor-general appoints ministers under advice from the prime minister, who is by convention the parliamentary leader of the governing party or coalition.[73] Cabinet, formed by ministers and led by the prime minister, is the highest policy-making body in government and responsible for deciding significant government actions.[74] Members of Cabinet make major decisions collectively, and are therefore collectively responsible for the consequences of these decisions.[75]

                    A parliamentary general election must be called no later than three years after the previous election.[76] Almost all general elections between 1853 and 1993 were held under the first-past-the-post voting system.[77] Since the 1996 election, a form of proportional representation called mixed-member proportional (MMP) has been used.[66] Under the MMP system, each person has two votes; one is for a candidate standing in the voter's electorate and the other is for a party. Since the 2014 election, there have been 71 electorates (which include seven Māori electorates in which only Māori can optionally vote),[78] and the remaining 49 of the 120 seats are assigned so that representation in parliament reflects the party vote, with the threshold that a party must win at least one electorate or 5% of the total party vote before it is eligible for a seat.[79]

                    A block of buildings fronted by a large statue.
                    A statue of Richard Seddon, the "Beehive" (Executive Wing), and Parliament House (right), in Parliament Grounds, Wellington.

                    Elections since the 1930s have been dominated by two political parties, National and Labour.[77] Between March 2005 and August 2006, New Zealand became the first country in the world in which all the highest offices in the land—head of state, governor-general, prime minister, speaker and chief justice—were occupied simultaneously by women.[80] The current prime minister is Jacinda Ardern, who has been in office since 26 October 2017.[81] She is the country's third female prime minister.[82]

                    New Zealand's judiciary, headed by the chief justice,[83] includes the Supreme Court, Court of Appeal, the High Court, and subordinate courts.[84] Judges and judicial officers are appointed non-politically and under strict rules regarding tenure to help maintain judicial independence.[66] This theoretically allows the judiciary to interpret the law based solely on the legislation enacted by Parliament without other influences on their decisions.[85]

                    New Zealand is identified as one of the world's most stable and well-governed states.[86] As at 2017, the country was ranked fourth in the strength of its democratic institutions,[87] and first in government transparency and lack of corruption.[88] A 2017 Human Rights Report by the U.S. Department of State noted that the government generally respected the rights of individuals, but voiced concerns regarding the social status of the Māori population.[89] New Zealand ranks highly for civic participation in the political process, with 77% voter turnout during recent elections, compared to an OECD average of 69%.[90]

                    See also: International rankings of New Zealand

                    Foreign relations and military

                    Main articles: Foreign relations of New Zealand and New Zealand Defence Force
                    A squad of men kneel in the desert sand while performing a war dance
                    Māori Battalion haka in Egypt, 1941

                    Early colonial New Zealand allowed the British Government to determine external trade and be responsible for foreign policy.[91] The 1923 and 1926 Imperial Conferences decided that New Zealand should be allowed to negotiate its own political treaties and the first commercial treaty was ratified in 1928 with Japan. On 3 September 1939 New Zealand allied itself with Britain and declared war on Germany with Prime Minister Michael Joseph Savage proclaiming, "Where she goes, we go; where she stands, we stand."[92]

                    In 1951 the United Kingdom became increasingly focused on its European interests,[93] while New Zealand joined Australia and the United States in the ANZUS security treaty.[94] The influence of the United States on New Zealand weakened following protests over the Vietnam War,[95] the refusal of the United States to admonish France after the sinking of the Rainbow Warrior,[96] disagreements over environmental and agricultural trade issues and New Zealand's nuclear-free policy.[97][98] Despite the United States' suspension of ANZUS obligations the treaty remained in effect between New Zealand and Australia, whose foreign policy has followed a similar historical trend.[99] Close political contact is maintained between the two countries, with free trade agreements and travel arrangements that allow citizens to visit, live and work in both countries without restrictions.[100] In 2013 there were about 650,000 New Zealand citizens living in Australia, which is equivalent to 15% of the resident population of New Zealand.[101]

                    A soldier in a green army uniform faces forwards
                    Anzac Day service at the National War Memorial

                    New Zealand has a strong presence among the Pacific Island countries. A large proportion of New Zealand's aid goes to these countries and many Pacific people migrate to New Zealand for employment.[102] Permanent migration is regulated under the 1970 Samoan Quota Scheme and the 2002 Pacific Access Category, which allow up to 1,100 Samoan nationals and up to 750 other Pacific Islanders respectively to become permanent New Zealand residents each year. A seasonal workers scheme for temporary migration was introduced in 2007 and in 2009 about 8,000 Pacific Islanders were employed under it.[103] A regional power,[104] New Zealand is involved in the Pacific Islands Forum, the Pacific Community, Asia-Pacific Economic Cooperation and the Association of Southeast Asian Nations Regional Forum (including the East Asia Summit).[100] New Zealand is a member of the United Nations,[105] the Commonwealth of Nations[106] and the Organisation for Economic Co-operation and Development (OECD),[107] and participates in the Five Power Defence Arrangements.[108]

                    New Zealand's military services—the Defence Force—comprise the New Zealand Army, the Royal New Zealand Air Force and the Royal New Zealand Navy.[109] New Zealand's national defence needs are modest, since a direct attack is unlikely.[110] However, its military has had a global presence. The country fought in both world wars, with notable campaigns in Gallipoli, Crete,[111] El Alamein[112] and Cassino.[113] The Gallipoli campaign played an important part in fostering New Zealand's national identity[114][115] and strengthened the ANZAC tradition it shares with Australia.[116]

                    In addition to Vietnam and the two world wars, New Zealand fought in the Second Boer War,[117] the Korean War,[118] the Malayan Emergency,[119] the Gulf War and the Afghanistan War. It has contributed forces to several regional and global peacekeeping missions, such as those in Cyprus, Somalia, Bosnia and Herzegovina, the Sinai, Angola, Cambodia, the Iran–Iraq border, Bougainville, East Timor, and the Solomon Islands.[120]

                    Local government and external territories

                    Main articles: Local government in New Zealand and Realm of New Zealand
                    Map with the North, South, Stewart/Rakiura, Tokelau, Cook, Niue, Kermadec, Chatham, Bounty, Antipodes, Snare, Auckland and Campbell Islands highlighted. New Zealand's segment of Antarctica (the Ross Dependency) is also highlighted.
                    Locations of the countries and territories within the Realm of New Zealand

                    The early European settlers divided New Zealand into provinces, which had a degree of autonomy.[121] Because of financial pressures and the desire to consolidate railways, education, land sales and other policies, government was centralised and the provinces were abolished in 1876.[122] The provinces are remembered in regional public holidays[123] and sporting rivalries.[124]

                    Since 1876, various councils have administered local areas under legislation determined by the central government.[121][125] In 1989, the government reorganised local government into the current two-tier structure of regional councils and territorial authorities.[126] The 249 municipalities[126] that existed in 1975 have now been consolidated into 67 territorial authorities and 11 regional councils.[127] The regional councils' role is to regulate "the natural environment with particular emphasis on resource management",[126] while territorial authorities are responsible for sewage, water, local roads, building consents and other local matters.[128][129] Five of the territorial councils are unitary authorities and also act as regional councils.[129] The territorial authorities consist of 13 city councils, 53 district councils, and the Chatham Islands Council. While officially the Chatham Islands Council is not a unitary authority, it undertakes many functions of a regional council.[130]

                    The Realm of New Zealand, one of 16 Commonwealth realms,[131] is the entire area over which the Queen of New Zealand is sovereign, and comprises New Zealand, Tokelau, the Ross Dependency, the Cook Islands and Niue.[65] The Cook Islands and Niue are self-governing states in free association with New Zealand.[132][133] The New Zealand Parliament cannot pass legislation for these countries, but with their consent can act on behalf of them in foreign affairs and defence. Tokelau is classified as a non-self-governing territory, but is administered by a council of three elders (one from each Tokelauan atoll).[134] The Ross Dependency is New Zealand's territorial claim in Antarctica, where it operates the Scott Base research facility.[135] New Zealand nationality law treats all parts of the realm equally, so most people born in New Zealand, the Cook Islands, Niue, Tokelau and the Ross Dependency are New Zealand citizens.[136][n 7]

                    Administrative divisions of the Realm of New Zealand
                    Countries  New Zealand      Cook Islands  Niue
                    Regions 11 non-unitary regions 5 unitary regions Chatham Islands   Outlying islands outside any regional authority
                    (the Kermadec Islands, Three Kings Islands, and Subantarctic Islands)                     		Ross Dependency  Tokelau 15 islands 14 villages
                    Territorial authorities 13 cities and 53 districts
                    Notes Some districts lie in more than one region These combine the regional and the territorial authority levels in one Special territorial authority The outlying Solander Islands form part of the Southland Region New Zealand's Antarctic territory Non-self-governing territory of New Zealand States in free association with New Zealand

                    Environment

                    Main article: Environment of New Zealand

                    Geography

                    Main articles: Geography of New Zealand and Geology of New Zealand
                    See also: Atlas of New Zealand at Wikimedia Commons
                    Islands of New Zealand as seen from satellite
                    The snow-capped Southern Alps dominate the South Island, while the North Island's Northland Peninsula stretches towards the subtropics

                    New Zealand is located near the centre of the water hemisphere and is made up of two main islands and a number of smaller islands. The two main islands (the North Island, or Te Ika-a-Māui, and the South Island, or Te Waipounamu) are separated by Cook Strait, 22 kilometres (14 mi) wide at its narrowest point.[138] Besides the North and South Islands, the five largest inhabited islands are Stewart Island (across the Foveaux Strait), Chatham Island, Great Barrier Island (in the Hauraki Gulf),[139] D'Urville Island (in the Marlborough Sounds)[140] and Waiheke Island (about 22 km (14 mi) from central Auckland).[141]

                    A large mountain with a lake in the foreground
                    Aoraki / Mount Cook is the highest point of New Zealand, at 3,754 metres
                    Snow-capped mountain range
                    The Southern Alps stretch for 500 kilometres down the South Island

                    New Zealand is long and narrow (over 1,600 kilometres (990 mi) along its north-north-east axis with a maximum width of 400 kilometres (250 mi)),[142] with about 15,000 km (9,300 mi) of coastline[143] and a total land area of 268,000 square kilometres (103,500 sq mi).[144] Because of its far-flung outlying islands and long coastline, the country has extensive marine resources. Its exclusive economic zone is one of the largest in the world, covering more than 15 times its land area.[145]

                    The South Island is the largest landmass of New Zealand. It is divided along its length by the Southern Alps.[146] There are 18 peaks over 3,000 metres (9,800 ft), the highest of which is Aoraki / Mount Cook at 3,754 metres (12,316 ft).[147] Fiordland's steep mountains and deep fiords record the extensive ice age glaciation of this southwestern corner of the South Island.[148] The North Island is less mountainous but is marked by volcanism.[149] The highly active Taupo Volcanic Zone has formed a large volcanic plateau, punctuated by the North Island's highest mountain, Mount Ruapehu (2,797 metres (9,177 ft)). The plateau also hosts the country's largest lake, Lake Taupo,[150] nestled in the caldera of one of the world's most active supervolcanoes.[151]

                    The country owes its varied topography, and perhaps even its emergence above the waves, to the dynamic boundary it straddles between the Pacific and Indo-Australian Plates.[152] New Zealand is part of Zealandia, a microcontinent nearly half the size of Australia that gradually submerged after breaking away from the Gondwanan supercontinent.[153] About 25 million years ago, a shift in plate tectonic movements began to contort and crumple the region. This is now most evident in the Southern Alps, formed by compression of the crust beside the Alpine Fault. Elsewhere the plate boundary involves the subduction of one plate under the other, producing the Puysegur Trench to the south, the Hikurangi Trench east of the North Island, and the Kermadec and Tonga Trenches[154] further north.[152]

                    New Zealand is part of a region known as Australasia, together with Australia.[155] It also forms the southwestern extremity of the geographic and ethnographic region called Polynesia.[156] The term Oceania is often used to denote the wider region encompassing the Australian continent, New Zealand and various islands in the Pacific Ocean that are not included in the seven-continent model.[157]

                    Climate

                    Main article: Climate of New Zealand

                    New Zealand's climate is predominantly temperate maritime (Köppen: Cfb), with mean annual temperatures ranging from 10 °C (50 °F) in the south to 16 °C (61 °F) in the north.[158] Historical maxima and minima are 42.4 °C (108.32 °F) in Rangiora, Canterbury and −25.6 °C (−14.08 °F) in Ranfurly, Otago.[159] Conditions vary sharply across regions from extremely wet on the West Coast of the South Island to almost semi-arid in Central Otago and the Mackenzie Basin of inland Canterbury and subtropical in Northland.[160] Of the seven largest cities, Christchurch is the driest, receiving on average only 640 millimetres (25 in) of rain per year and Wellington the wettest, receiving almost twice that amount.[161] Auckland, Wellington and Christchurch all receive a yearly average of more than 2,000 hours of sunshine. The southern and southwestern parts of the South Island have a cooler and cloudier climate, with around 1,400–1,600 hours; the northern and northeastern parts of the South Island are the sunniest areas of the country and receive about 2,400–2,500 hours.[162] The general snow season is early June until early October, though cold snaps can occur outside this season.[163] Snowfall is common in the eastern and southern parts of the South Island and mountain areas across the country.[158]

                    The table below lists climate normals for the warmest and coldest months in New Zealand's six largest cities. North Island cities are generally warmest in February. South Island cities are warmest in January.

                    Average daily maximum and minimum temperatures for the six largest cities of New Zealand[164]
                    Location Jan/Feb (°C) Jan/Feb (°F) July (°C) July (°F)
                    Auckland 23/16 74/60 14/7 58/45
                    Wellington 20/13 68/56 11/6 52/42
                    Christchurch 22/12 72/53 10/0 51/33
                    Hamilton 24/13 75/56 14/4 57/39
                    Tauranga 24/15 75/59 14/6 58/42
                    Dunedin 19/11 66/53 10/3 50/37

                    Biodiversity

                    Main article: Biodiversity of New Zealand
                    Kiwi amongst sticks
                    The endemic flightless kiwi is a national icon.

                    New Zealand's geographic isolation for 80 million years[165] and island biogeography has influenced evolution of the country's species of animals, fungi and plants. Physical isolation has caused biological isolation, resulting in a dynamic evolutionary ecology with examples of very distinctive plants and animals as well as populations of widespread species.[166][167] About 82% of New Zealand's indigenous vascular plants are endemic, covering 1,944 species across 65 genera.[168][169] The number of fungi recorded from New Zealand, including lichen-forming species, is not known, nor is the proportion of those fungi which are endemic, but one estimate suggests there are about 2,300 species of lichen-forming fungi in New Zealand[168] and 40% of these are endemic.[170] The two main types of forest are those dominated by broadleaf trees with emergent podocarps, or by southern beech in cooler climates.[171] The remaining vegetation types consist of grasslands, the majority of which are tussock.[172]

                    Before the arrival of humans, an estimated 80% of the land was covered in forest, with only high alpine, wet, infertile and volcanic areas without trees.[173] Massive deforestation occurred after humans arrived, with around half the forest cover lost to fire after Polynesian settlement.[174] Much of the remaining forest fell after European settlement, being logged or cleared to make room for pastoral farming, leaving forest occupying only 23% of the land.[175]

                    An artist's rendition of a Haast's eagle attacking two moa
                    The giant Haast's eagle died out when humans hunted its main prey, the moa, to extinction.

                    The forests were dominated by birds, and the lack of mammalian predators led to some like the kiwi, kakapo, weka and takahē evolving flightlessness.[176] The arrival of humans, associated changes to habitat, and the introduction of rats, ferrets and other mammals led to the extinction of many bird species, including large birds like the moa and Haast's eagle.[177][178]

                    Other indigenous animals are represented by reptiles (tuatara, skinks and geckos), frogs,[179] spiders,[180] insects (weta)[181] and snails.[182] Some, such as the tuatara, are so unique that they have been called living fossils.[183] Three species of bats (one since extinct) were the only sign of native land mammals in New Zealand until the 2006 discovery of bones from a unique, mouse-sized land mammal at least 16 million years old.[184][185] Marine mammals however are abundant, with almost half the world's cetaceans (whales, dolphins, and porpoises) and large numbers of fur seals reported in New Zealand waters.[186] Many seabirds breed in New Zealand, a third of them unique to the country.[187] More penguin species are found in New Zealand than in any other country.[188]

                    Since human arrival, almost half of the country's vertebrate species have become extinct, including at least fifty-one birds, three frogs, three lizards, one freshwater fish, and one bat. Others are endangered or have had their range severely reduced.[177] However, New Zealand conservationists have pioneered several methods to help threatened wildlife recover, including island sanctuaries, pest control, wildlife translocation, fostering, and ecological restoration of islands and other selected areas.[189][190][191][192]

                    Economy

                    Main article: Economy of New Zealand
                    See also: List of companies of New Zealand
                    Boats docked in blue-green water. Plate glass skyscrapers rising up in the background.
                    Waterfront along Auckland CBD, a major hub of economic activity

                    New Zealand has an advanced market economy,[193] ranked 16th in the 2018 Human Development Index[8] and third in the 2018 Index of Economic Freedom.[194] It is a high-income economy with a nominal gross domestic product (GDP) per capita of US$36,254.[6] The currency is the New Zealand dollar, informally known as the "Kiwi dollar"; it also circulates in the Cook Islands (see Cook Islands dollar), Niue, Tokelau, and the Pitcairn Islands.[195]

                    Historically, extractive industries have contributed strongly to New Zealand's economy, focussing at different times on sealing, whaling, flax, gold, kauri gum, and native timber.[196] The first shipment of refrigerated meat on the Dunedin in 1882 led to the establishment of meat and dairy exports to Britain, a trade which provided the basis for strong economic growth in New Zealand.[197] High demand for agricultural products from the United Kingdom and the United States helped New Zealanders achieve higher living standards than both Australia and Western Europe in the 1950s and 1960s.[198] In 1973, New Zealand's export market was reduced when the United Kingdom joined the European Economic Community[199] and other compounding factors, such as the 1973 oil and 1979 energy crises, led to a severe economic depression.[200] Living standards in New Zealand fell behind those of Australia and Western Europe, and by 1982 New Zealand had the lowest per-capita income of all the developed nations surveyed by the World Bank.[201] In the mid-1980s New Zealand deregulated its agricultural sector by phasing out subsidies over a three-year period.[202][203] Since 1984, successive governments engaged in major macroeconomic restructuring (known first as Rogernomics and then Ruthanasia), rapidly transforming New Zealand from a protected and highly regulated economy to a liberalised free-trade economy.[204][205]

                    Blue water against a backdrop of snow-capped mountains
                    Milford Sound is one of New Zealand's most famous tourist destinations.[206]

                    Unemployment peaked above 10% in 1991 and 1992,[207] following the 1987 share market crash, but eventually fell to a record low (since 1986) of 3.7% in 2007 (ranking third from twenty-seven comparable OECD nations).[207] However, the global financial crisis that followed had a major impact on New Zealand, with the GDP shrinking for five consecutive quarters, the longest recession in over thirty years,[208][209] and unemployment rising back to 7% in late 2009.[210] Unemployment rates for different age groups follow similar trends, but are consistently higher among youth. In the December 2014 quarter, the general unemployment rate was around 5.8%, while the unemployment rate for youth aged 15 to 21 was 15.6%.[207] New Zealand has experienced a series of "brain drains" since the 1970s[211] that still continue today.[212] Nearly one quarter of highly skilled workers live overseas, mostly in Australia and Britain, which is the largest proportion from any developed nation.[213] In recent decades, however, a "brain gain" has brought in educated professionals from Europe and less developed countries.[214][215] Today New Zealand's economy benefits from a high level of innovation.[216]

                    Trade

                    New Zealand is heavily dependent on international trade,[217] particularly in agricultural products.[218] Exports account for 24% of its output,[143] making New Zealand vulnerable to international commodity prices and global economic slowdowns. Food products made up 55% of the value of all the country's exports in 2014; wood was the second largest earner (7%).[219] New Zealand's main trading partners, as at June 2018, are China (NZ$27.8b), Australia ($26.2b), the European Union ($22.9b), the United States ($17.6b), and Japan ($8.4b).[220] On 7 April 2008, New Zealand and China signed the New Zealand–China Free Trade Agreement, the first such agreement China has signed with a developed country.[221] The service sector is the largest sector in the economy, followed by manufacturing and construction and then farming and raw material extraction.[143] Tourism plays a significant role in the economy, contributing $12.9 billion (or 5.6%) to New Zealand's total GDP and supporting 7.5% of the total workforce in 2016.[222] International visitor arrivals are expected to increase at a rate of 5.4% annually up to 2022.[222]

                    A Romney ewe with her two lambs
                    Wool has historically been one of New Zealand's major exports.

                    Wool was New Zealand's major agricultural export during the late 19th century.[196] Even as late as the 1960s it made up over a third of all export revenues,[196] but since then its price has steadily dropped relative to other commodities[223] and wool is no longer profitable for many farmers.[224] In contrast dairy farming increased, with the number of dairy cows doubling between 1990 and 2007,[225] to become New Zealand's largest export earner.[226] In the year to June 2018, dairy products accounted for 17.7% ($14.1 billion) of total exports,[220] and the country's largest company, Fonterra, controls almost one-third of the international dairy trade.[227] Other exports in 2017-18 were meat (8.8%), wood and wood products (6.2%), fruit (3.6%), machinery (2.2%) and wine (2.1%).[220] New Zealand's wine industry has followed a similar trend to dairy, the number of vineyards doubling over the same period,[228] overtaking wool exports for the first time in 2007.[229][230]

                    Infrastructure

                    A mid-size jet airliner in flight. The plane livery is all-black and features a New Zealand silver fern mark.
                    A Boeing 787-9 Dreamliner of Air New Zealand, the flag carrier of New Zealand

                    In 2015, renewable energy, primarily geothermal and hydroelectric power, generated 40.1% of New Zealand's gross energy supply.[231] Geothermal power alone accounted for 22% of New Zealand's energy in 2015.[231]

                    The provision of water supply and sanitation is generally of good quality. Regional authorities provide water abstraction, treatment and distribution infrastructure to most developed areas.[232][233]

                    New Zealand's transport network comprises 94,000 kilometres (58,410 mi) of roads, including 199 kilometres (124 mi) of motorways,[234] and 4,128 kilometres (2,565 mi) of railway lines.[143] Most major cities and towns are linked by bus services, although the private car is the predominant mode of transport.[235] The railways were privatised in 1993, but were re-nationalised by the government in stages between 2004 and 2008. The state-owned enterprise KiwiRail now operates the railways, with the exception of commuter services in Auckland and Wellington which are operated by Transdev[236] and Metlink,[237] respectively. Railways run the length of the country, although most lines now carry freight rather than passengers.[238] Most international visitors arrive via air[239] and New Zealand has six international airports, but currently only the Auckland and Christchurch airports connect directly with countries other than Australia or Fiji.[240]

                    The New Zealand Post Office had a monopoly over telecommunications in New Zealand until 1987 when Telecom New Zealand was formed, initially as a state-owned enterprise and then privatised in 1990.[241] Chorus, which was split from Telecom (now Spark) in 2011,[242] still owns the majority of the telecommunications infrastructure, but competition from other providers has increased.[241] A large-scale rollout of gigabit-capable fibre to the premises, branded as Ultra-Fast Broadband, began in 2009 with a target of being available to 87% of the population by 2022.[243] As of 2017, the United Nations International Telecommunication Union ranks New Zealand 13th in the development of information and communications infrastructure.[244]

                    Demography

                    Main article: Demographics of New Zealand
                    Stationary population pyramid broken down into 21 age ranges.
                    Population pyramid (2017)

                    The 2013 New Zealand census enumerated a resident population of 4,242,048, an increase of 5.3% over the 2006 figure.[245][n 8] As of September 2019, the total population has risen to an estimated 4,933,210.[5]

                    New Zealand is a predominantly urban country, with 73.0% of the population living in the seventeen main urban areas (i.e. population 30,000 or greater) and 55.1% living in the four largest cities of Auckland, Christchurch, Wellington, and Hamilton.[247] New Zealand cities generally rank highly on international livability measures. For instance, in 2016 Auckland was ranked the world's third most liveable city and Wellington the twelfth by the Mercer Quality of Living Survey.[248]

                    Life expectancy for New Zealanders in 2012 was 84 years for females, and 80.2 years for males.[249] Life expectancy at birth is forecast to increase from 80 years to 85 years in 2050 and infant mortality is expected to decline.[250] New Zealand's fertility rate of 2.1 is relatively high for a developed country, and natural births account for a significant proportion of population growth. Consequently, the country has a young population compared to most industrialised nations, with 20% of New Zealanders being 14 years old or younger.[143] In 2018 the median age of the New Zealand population was 38.1 years.[251] By 2050 the median age is projected to rise to 43 years and the percentage of people 60 years of age and older to rise from 18% to 29%.[250] In 2008 the leading cause of premature death was cancer, at 29.8%, followed by ischaemic heart disease, 19.7%, and then cerebrovascular disease, 9.2%.[252] As of 2016, total expenditure on health care (including private sector spending) is 9.2% of GDP.[253]

                     
                    Largest urban areas in New Zealand
                    Statistics New Zealand June 2018 estimate (NZSAC92 boundaries)[254]
                    Rank Name Region Pop. Rank Name Region Pop.
                    Auckland
                    Auckland
                    Wellington
                    Wellington                     		1 Auckland Auckland 1,628,900 11 Whangarei Northland 58,800 Christchurch
                    Christchurch
                    Hamilton
                    Hamilton
                    2 Wellington Wellington 418,500 12 New Plymouth Taranaki 58,300
                    3 Christchurch Canterbury 404,500 13 Invercargill Southland 51,200
                    4 Hamilton Waikato 241,200 14 Kapiti Wellington 42,700
                    5 Tauranga Bay of Plenty 141,600 15 Whanganui Manawatu-Wanganui 40,900
                    6 Napier-Hastings Hawke's Bay 134,500 16 Gisborne Gisborne 37,200
                    7 Dunedin Otago 122,000 17 Blenheim Marlborough 31,600
                    8 Palmerston North Manawatu-Wanganui 86,600 18 Pukekohe Auckland 31,400
                    9 Nelson Nelson 67,500 19 Timaru Canterbury 29,100
                    10 Rotorua Bay of Plenty 59,500 20 Taupo Waikato 24,700


                    Ethnicity and immigration

                    Main articles: New Zealanders and Immigration to New Zealand
                    Pedestrians crossing a wide street which is flanked by storefronts
                    Pedestrians on Queen Street in Auckland, an ethnically diverse city

                    In the 2013 census, 74.0% of New Zealand residents identified ethnically as European, and 14.9% as Māori. Other major ethnic groups include Asian (11.8%) and Pacific peoples (7.4%), two-thirds of whom live in the Auckland Region.[255][n 3] The population has become more diverse in recent decades: in 1961, the census reported that the population of New Zealand was 92% European and 7% Māori, with Asian and Pacific minorities sharing the remaining 1%.[256]

                    While the demonym for a New Zealand citizen is New Zealander, the informal "Kiwi" is commonly used both internationally[257] and by locals.[258] The Māori loanword Pākehā has been used to refer to New Zealanders of European descent, although others reject this appellation.[259][260] The word Pākehā today is increasingly used to refer to all non-Polynesian New Zealanders.[261]

                    The Māori were the first people to reach New Zealand, followed by the early European settlers. Following colonisation, immigrants were predominantly from Britain, Ireland and Australia because of restrictive policies similar to the White Australia policy.[262] There was also significant Dutch, Dalmatian,[263] German, and Italian immigration, together with indirect European immigration through Australia, North America, South America and South Africa.[264][265] Net migration increased after the Second World War; in the 1970s and 1980s policies were relaxed and immigration from Asia was promoted.[265][266] In 2009–10, an annual target of 45,000–50,000 permanent residence approvals was set by the New Zealand Immigration Service—more than one new migrant for every 100 New Zealand residents.[267] Just over 25% of New Zealand's population was born overseas, with the majority (52%) living in the Auckland Region. The United Kingdom remains the largest source of New Zealand's overseas population, with a quarter of all overseas-born New Zealanders born there; other major sources of New Zealand's overseas-born population are China, India, Australia, South Africa, Fiji and Samoa.[268] The number of fee-paying international students increased sharply in the late 1990s, with more than 20,000 studying in public tertiary institutions in 2002.[269]

                    Language

                    Main article: Languages of New Zealand
                    Map of New Zealand showing the percentage of people in each census area unit who speak Māori. Areas of the North Island exhibit the highest Māori proficiency.
                    Speakers of Māori according to the 2013 census[270]
                      Less than 5%
                      More than 5%
                      More than 10%
                      More than 20%
                      More than 30%
                      More than 40%
                      More than 50%

                    English is the predominant language in New Zealand, spoken by 96.1% of the population.[271] New Zealand English is similar to Australian English and many speakers from the Northern Hemisphere are unable to tell the accents apart.[272] The most prominent differences between the New Zealand English dialect and other English dialects are the shifts in the short front vowels: the short-"i" sound (as in "kit") has centralised towards the schwa sound (the "a" in "comma" and "about"); the short-"e" sound (as in "dress") has moved towards the short-"i" sound; and the short-"a" sound (as in "trap") has moved to the short-"e" sound.[273]

                    After the Second World War, Māori were discouraged from speaking their own language (te reo Māori) in schools and workplaces and it existed as a community language only in a few remote areas.[274] It has recently undergone a process of revitalisation,[275] being declared one of New Zealand's official languages in 1987,[276] and is spoken by 3.7% of the population.[271][n 9] There are now Māori language immersion schools and two television channels that broadcast predominantly in Māori.[278] Many places have both their Māori and English names officially recognised.[279]

                    As recorded in the 2013 census,[271] Samoan is the most widely spoken non-official language (2.2%),[n 10] followed by Hindi (1.7%), "Northern Chinese" (including Mandarin, 1.3%) and French (1.2%). 20,235 people (0.5%) reported the ability to use New Zealand Sign Language. It was declared one of New Zealand's official languages in 2006.[280]

                    Religion

                    Main article: Religion in New Zealand
                    See also: Irreligion in New Zealand
                    Simple white building with two red domed towers
                    A Rātana church on a hill near Raetihi. The two-tower construction is characteristic of Rātana buildings.

                    Christianity is the predominant religion in New Zealand, although its society is among the most secular in the world.[281][282] In the 2018 census, 51.4% of the population identified with one or more religions, including 38.6% identifying as Christians. Another 48.6% indicated that they had no religion.[n 11][283] The main Christian denominations are, by number of adherents, Roman Catholicism (10.1%), Anglicanism (6.8%), Presbyterianism (5.5%) and "Christian not further defined" (i.e. people identifying as Christian but not stating the denomination, 6.6%).[283] The Māori-based Ringatū and Rātana religions (1.3%) are also Christian in origin.[284][285] Immigration and demographic change in recent decades has contributed to the growth of minority religions,[286] such as Hinduism (2.6%), Buddhism (1.1%), Islam (1.3%) and Sikhism (0.5%).[284] The Auckland Region exhibited the greatest religious diversity.[284]

                    Education

                    Main articles: Education in New Zealand and Tertiary education in New Zealand

                    Primary and secondary schooling is compulsory for children aged 6 to 16, with the majority attending from the age of 5.[287] There are 13 school years and attending state (public) schools is free to New Zealand citizens and permanent residents from a person's 5th birthday to the end of the calendar year following their 19th birthday.[288] New Zealand has an adult literacy rate of 99%,[143] and over half of the population aged 15 to 29 hold a tertiary qualification.[287] There are five types of government-owned tertiary institutions: universities, colleges of education, polytechnics, specialist colleges, and wānanga,[289] in addition to private training establishments.[290] In the adult population 14.2% have a bachelor's degree or higher, 30.4% have some form of secondary qualification as their highest qualification and 22.4% have no formal qualification.[291] The OECD's Programme for International Student Assessment ranks New Zealand's education system as the seventh best in the world, with students performing exceptionally well in reading, mathematics and science.[292]

                    Culture

                    Main article: Culture of New Zealand
                    Tall wooden carving showing Kupe above two tentacled sea creatures
                    Late 20th-century house-post depicting the navigator Kupe fighting two sea creatures

                    Early Māori adapted the tropically based east Polynesian culture in line with the challenges associated with a larger and more diverse environment, eventually developing their own distinctive culture. Social organisation was largely communal with families (whānau), subtribes (hapū) and tribes (iwi) ruled by a chief (rangatira), whose position was subject to the community's approval.[293] The British and Irish immigrants brought aspects of their own culture to New Zealand and also influenced Māori culture,[294][295] particularly with the introduction of Christianity.[296] However, Māori still regard their allegiance to tribal groups as a vital part of their identity, and Māori kinship roles resemble those of other Polynesian peoples.[297] More recently American, Australian, Asian and other European cultures have exerted influence on New Zealand. Non-Māori Polynesian cultures are also apparent, with Pasifika, the world's largest Polynesian festival, now an annual event in Auckland.[298]

                    The largely rural life in early New Zealand led to the image of New Zealanders being rugged, industrious problem solvers.[299] Modesty was expected and enforced through the "tall poppy syndrome", where high achievers received harsh criticism.[300] At the time New Zealand was not known as an intellectual country.[301] From the early 20th century until the late 1960s, Māori culture was suppressed by the attempted assimilation of Māori into British New Zealanders.[274] In the 1960s, as tertiary education became more available and cities expanded[302] urban culture began to dominate.[303] However, rural imagery and themes are common in New Zealand's art, literature and media.[304]

                    New Zealand's national symbols are influenced by natural, historical, and Māori sources. The silver fern is an emblem appearing on army insignia and sporting team uniforms.[305] Certain items of popular culture thought to be unique to New Zealand are called "Kiwiana".[305]

                    Art

                    Main article: New Zealand art

                    As part of the resurgence of Māori culture, the traditional crafts of carving and weaving are now more widely practised and Māori artists are increasing in number and influence.[306] Most Māori carvings feature human figures, generally with three fingers and either a natural-looking, detailed head or a grotesque head.[307] Surface patterns consisting of spirals, ridges, notches and fish scales decorate most carvings.[308] The pre-eminent Māori architecture consisted of carved meeting houses (wharenui) decorated with symbolic carvings and illustrations. These buildings were originally designed to be constantly rebuilt, changing and adapting to different whims or needs.[309]

                    Māori decorated the white wood of buildings, canoes and cenotaphs using red (a mixture of red ochre and shark fat) and black (made from soot) paint and painted pictures of birds, reptiles and other designs on cave walls.[310] Māori tattoos (moko) consisting of coloured soot mixed with gum were cut into the flesh with a bone chisel.[311] Since European arrival paintings and photographs have been dominated by landscapes, originally not as works of art but as factual portrayals of New Zealand.[312] Portraits of Māori were also common, with early painters often portraying them as "noble savages", exotic beauties or friendly natives.[312] The country's isolation delayed the influence of European artistic trends allowing local artists to develop their own distinctive style of regionalism.[313] During the 1960s and 1970s many artists combined traditional Māori and Western techniques, creating unique art forms.[314] New Zealand art and craft has gradually achieved an international audience, with exhibitions in the Venice Biennale in 2001 and the "Paradise Now" exhibition in New York in 2004.[306][315]

                    Refer to caption
                    Portrait of Hinepare of Ngāti Kahungunu by Gottfried Lindauer, showing chin moko, pounamu hei-tiki and woven cloak

                    Māori cloaks are made of fine flax fibre and patterned with black, red and white triangles, diamonds and other geometric shapes.[316] Greenstone was fashioned into earrings and necklaces, with the most well-known design being the hei-tiki, a distorted human figure sitting cross-legged with its head tilted to the side.[317] Europeans brought English fashion etiquette to New Zealand, and until the 1950s most people dressed up for social occasions.[318] Standards have since relaxed and New Zealand fashion has received a reputation for being casual, practical and lacklustre.[319][320] However, the local fashion industry has grown significantly since 2000, doubling exports and increasing from a handful to about 50 established labels, with some labels gaining international recognition.[320]

                    Literature

                    Main article: New Zealand literature

                    Māori quickly adopted writing as a means of sharing ideas, and many of their oral stories and poems were converted to the written form.[321] Most early English literature was obtained from Britain and it was not until the 1950s when local publishing outlets increased that New Zealand literature started to become widely known.[322] Although still largely influenced by global trends (modernism) and events (the Great Depression), writers in the 1930s began to develop stories increasingly focused on their experiences in New Zealand. During this period literature changed from a journalistic activity to a more academic pursuit.[323] Participation in the world wars gave some New Zealand writers a new perspective on New Zealand culture and with the post-war expansion of universities local literature flourished.[324] Dunedin is a UNESCO City of Literature.[325]

                    Media and entertainment

                    Main articles: Music of New Zealand, Cinema of New Zealand, and Media of New Zealand

                    New Zealand music has been influenced by blues, jazz, country, rock and roll and hip hop, with many of these genres given a unique New Zealand interpretation.[326] Māori developed traditional chants and songs from their ancient Southeast Asian origins, and after centuries of isolation created a unique "monotonous" and "doleful" sound.[327] Flutes and trumpets were used as musical instruments[328] or as signalling devices during war or special occasions.[329] Early settlers brought over their ethnic music, with brass bands and choral music being popular, and musicians began touring New Zealand in the 1860s.[330][331] Pipe bands became widespread during the early 20th century.[332] The New Zealand recording industry began to develop from 1940 onwards and many New Zealand musicians have obtained success in Britain and the United States.[326] Some artists release Māori language songs and the Māori tradition-based art of kapa haka (song and dance) has made a resurgence.[333] The New Zealand Music Awards are held annually by Recorded Music NZ; the awards were first held in 1965 by Reckitt & Colman as the Loxene Golden Disc awards.[334] Recorded Music NZ also publishes the country's official weekly record charts.[335]

                    Hills with inset, round doors. Reflected in water.
                    The Hobbiton Movie Set, located near Matamata, was used for The Lord of the Rings film trilogy.[336]

                    Public radio was introduced in New Zealand in 1922.[337] A state-owned television service began in 1960.[338] Deregulation in the 1980s saw a sudden increase in the numbers of radio and television stations.[339] New Zealand television primarily broadcasts American and British programming, along with a large number of Australian and local shows.[340] The number of New Zealand films significantly increased during the 1970s. In 1978 the New Zealand Film Commission started assisting local film-makers and many films attained a world audience, some receiving international acknowledgement.[339] The highest-grossing New Zealand films are Hunt for the Wilderpeople, Boy, The World's Fastest Indian, Once Were Warriors and Whale Rider.[341] The country's diverse scenery and compact size, plus government incentives,[342] have encouraged some producers to shoot big-budget productions in New Zealand, including Avatar, The Lord of the Rings, The Hobbit, The Chronicles of Narnia, King Kong and The Last Samurai.[343] The New Zealand media industry is dominated by a small number of companies, most of which are foreign-owned, although the state retains ownership of some television and radio stations.[344] Since 1994, Freedom House has consistently ranked New Zealand's press freedom in the top twenty, with the 19th freest media in 2015.[345]

                    Sports

                    Main article: Sport in New Zealand
                    Rugby team wearing all black, facing the camera, knees bent, and facing toward a team wearing white
                    A haka performed by the national rugby union team before a game. The haka is a challenge with vigorous movements and stamping of the feet.

                    Most of the major sporting codes played in New Zealand have British origins.[346] Rugby union is considered the national sport[347] and attracts the most spectators.[348] Golf, netball, tennis and cricket have the highest rates of adult participation, while netball, rugby union and football (soccer) are particularly popular among young people.[348][349] Around 54% of New Zealand adolescents participate in sports for their school.[349] Victorious rugby tours to Australia and the United Kingdom in the late 1880s and the early 1900s played an early role in instilling a national identity.[350] Horseracing was also a popular spectator sport and became part of the "Rugby, Racing and Beer" culture during the 1960s.[351] Māori participation in European sports was particularly evident in rugby and the country's team performs a haka, a traditional Māori challenge, before international matches.[352] New Zealand is known for its extreme sports, adventure tourism[353] and strong mountaineering tradition, as seen in the success of notable New Zealander Sir Edmund Hillary.[354][355] Other outdoor pursuits such as cycling, fishing, swimming, running, tramping, canoeing, hunting, snowsports, surfing and sailing are also popular.[356] The Polynesian sport of waka ama racing has experienced a resurgence of interest in New Zealand since the 1980s.[357]

                    New Zealand has competitive international teams in rugby union, rugby league, netball, cricket, softball, and sailing. New Zealand participated at the Summer Olympics in 1908 and 1912 as a joint team with Australia, before first participating on its own in 1920.[358] The country has ranked highly on a medals-to-population ratio at recent Games.[359][360] The "All Blacks", the national rugby union team, are the most successful in the history of international rugby[361] and the reigning World Cup champions.[362]

                    Cuisine

                    Main article: New Zealand cuisine
                    Raw meat and vegetables
                    Ingredients to be prepared for a hāngi

                    The national cuisine has been described as Pacific Rim, incorporating the native Māori cuisine and diverse culinary traditions introduced by settlers and immigrants from Europe, Polynesia and Asia.[363] New Zealand yields produce from land and sea—most crops and livestock, such as maize, potatoes and pigs, were gradually introduced by the early European settlers.[364] Distinctive ingredients or dishes include lamb, salmon, kōura (crayfish),[365] dredge oysters, whitebait, pāua (abalone), mussels, scallops, pipis and tuatua (both are types of New Zealand shellfish),[366] kūmara (sweet potato), kiwifruit, tamarillo and pavlova (considered a national dish).[367][363] A hāngi is a traditional Māori method of cooking food using heated rocks buried in a pit oven. After European colonisation, Māori began cooking with pots and ovens and the hāngi was used less frequently, although it is still used for formal occasions such as tangihanga.[368]

                    See also

                    Footnotes

                    1. ^ "God Save the Queen" is officially a national anthem but is generally used only on regal and viceregal occasions.[1]
                    2. ^ English is a de facto official language due to its widespread use.[2]
                    3. ^ a b Ethnicity figures add to more than 100% as people could choose more than one ethnic group.
                    4. ^ The proportion of New Zealand's area (excluding estuaries) covered by rivers, lakes and ponds, based on figures from the New Zealand Land Cover Database,[4] is (357526 + 81936) / (26821559 – 92499–26033 – 19216) = 1.6%. If estuarine open water, mangroves, and herbaceous saline vegetation are included, the figure is 2.2%.
                    5. ^ The Chatham Islands have a separate time zone, 45 minutes ahead of the rest of New Zealand.
                    6. ^ Clocks are advanced by an hour from the last Sunday in September until the first Sunday in April.[9] Daylight saving time is also observed in the Chatham Islands, 45 minutes ahead of NZDT.
                    7. ^ A person born on or after 1 January 2006 acquires New Zealand citizenship at birth only if at least one parent is a New Zealand citizen or permanent resident. People born on or before 31 December 2005 acquired citizenship at birth (jus soli).[137]
                    8. ^ The population is increasing at a rate of 1.4–2.0% per year and is projected to rise to 5.01–5.51 million in 2025.[246]
                    9. ^ In 2015, 55% of Māori adults (aged 15 years and over) reported knowledge of te reo Māori. Of these speakers, 64% use Māori at home and 50,000 can speak the language "very well" or "well".[277]
                    10. ^ Of the 86,403 people that replied they spoke Samoan, 51,336 lived in the Auckland Region.
                    11. ^ Religion percentages may not add to 100% as people could claim multiple religions or object to answering the question.

                    Citations

                    1. ^ "Protocol for using New Zealand's National Anthems". Ministry for Culture and Heritage. Retrieved 17 February 2008.
                    2. ^ New Zealand Government (21 December 2007). International Covenant on Civil and Political Rights Fifth Periodic Report of the Government of New Zealand (PDF) (Report). p. 89. Archived from the original (PDF) on 24 January 2015. Retrieved 18 November 2015. In addition to the Māori language, New Zealand Sign Language is also an official language of New Zealand. The New Zealand Sign Language Act 2006 permits the use of NZSL in legal proceedings, facilitates competency standards for its interpretation and guides government departments in its promotion and use. English, the medium for teaching and learning in most schools, is a de facto official language by virtue of its widespread use. For these reasons, these three languages have special mention in the New Zealand Curriculum.
                    3. ^ "2018 Census population and dwelling counts". Statistics New Zealand. Retrieved 26 September 2019.
                    4. ^ "The New Zealand Land Cover Database". New Zealand Land Cover Database 2. Ministry for the Environment. 1 July 2009. Retrieved 26 April 2011.
                    5. ^ a b "Population clock". Statistics New Zealand. Retrieved 14 April 2016. The population estimate shown is automatically calculated daily at 00:00 UTC and is based on data obtained from the population clock on the date shown in the citation.
                    6. ^ a b c d e "New Zealand". International Monetary Fund. Retrieved 9 October 2018.
                    7. ^ "Income inequality". Statistics New Zealand. Retrieved 14 June 2015.
                    8. ^ a b "Human Development Report 2018" (PDF). HDRO (Human Development Report Office) United Nations Development Programme. p. 22. Retrieved 14 September 2018.
                    9. ^ "New Zealand Daylight Time Order 2007 (SR 2007/185)". New Zealand Parliamentary Counsel Office. 6 July 2007. Retrieved 6 March 2017.
                    10. ^ There is no official all-numeric date format for New Zealand, but government recommendations generally follow Australian date and time notation. See "The Govt.nz style guide", New Zealand Government, 9 December 2016, retrieved 7 March 2019 Italic or bold markup not allowed in: |website= (help) .
                    11. ^ Tasman, Abel. "JOURNAL or DESCRIPTION By me Abel Jansz Tasman, Of a Voyage from Batavia for making Discoveries of the Unknown South Land in the year 1642". Project Gutenberg Australia. Retrieved 26 March 2018.
                    12. ^ Wilson, John (March 2009). "European discovery of New Zealand – Tasman's achievement". Te Ara: The Encyclopedia of New Zealand. Retrieved 24 January 2011.
                    13. ^ John Bathgate. "The Pamphlet Collection of Sir Robert Stout:Volume 44. Chapter 1, Discovery and Settlement". NZETC. Retrieved 17 August 2018. He named the country Staaten Land, in honour of the States-General of Holland, in the belief that it was part of the great southern continent.
                    14. ^ Wilson, John (September 2007). "Tasman's achievement". Te Ara: The Encyclopedia of New Zealand. Retrieved 16 February 2008.
                    15. ^ Mackay, Duncan (1986). "The Search For The Southern Land". In Fraser, B (ed.). The New Zealand Book Of Events. Auckland: Reed Methuen. pp. 52–54.
                    16. ^ a b McKinnon, Malcolm (November 2009). "Place names – Naming the country and the main islands". Te Ara: The Encyclopedia of New Zealand. Retrieved 24 January 2011.
                    17. ^ King 2003, p. 41.
                    18. ^ Hay, Maclagan & Gordon 2008, p. 72.
                    19. ^ a b Mein Smith 2005, p. 6.
                    20. ^ Brunner, Thomas (1851). The Great Journey: an expedition to explore the interior of the Middle Island, New Zealand, 1846-8. Royal Geographical Society.
                    21. ^ a b Williamson, Maurice (10 October 2013). "Names of NZ's two main islands formalised" (Press release). New Zealand Government. Retrieved 1 May 2017.
                    22. ^ Wilmshurst, J. M.; Hunt, T. L.; Lipo, C. P.; Anderson, A. J. (2010). "High-precision radiocarbon dating shows recent and rapid initial human colonization of East Polynesia". Proceedings of the National Academy of Sciences. 108 (5): 1815. Bibcode:2011PNAS..108.1815W. doi:10.1073/pnas.1015876108. PMC 3033267. PMID 21187404.
                    23. ^ McGlone, M.; Wilmshurst, J. M. (1999). "Dating initial Maori environmental impact in New Zealand". Quaternary International. 59: 5–16. Bibcode:1999QuInt..59....5M. doi:10.1016/S1040-6182(98)00067-6.
                    24. ^ Murray-McIntosh, Rosalind P.; Scrimshaw, Brian J.; Hatfield, Peter J.; Penny, David (1998). "Testing migration patterns and estimating founding population size in Polynesia by using human mtDNA sequences". Proceedings of the National Academy of Sciences of the United States of America. 95 (15): 9047–52. Bibcode:1998PNAS...95.9047M. doi:10.1073/pnas.95.15.9047. PMC 21200.
                    25. ^ Wilmshurst, J. M.; Anderson, A. J.; Higham, T. F. G.; Worthy, T. H. (2008). "Dating the late prehistoric dispersal of Polynesians to New Zealand using the commensal Pacific rat". Proceedings of the National Academy of Sciences. 105 (22): 7676. Bibcode:2008PNAS..105.7676W. doi:10.1073/pnas.0801507105. PMC 2409139. PMID 18523023.
                    26. ^ Moodley, Y.; Linz, B.; Yamaoka, Y.; Windsor, H.M.; Breurec, S.; Wu, J.-Y.; Maady, A.; Bernhöft, S.; Thiberge, J.-M.; et al. (2009). "The Peopling of the Pacific from a Bacterial Perspective". Science. 323 (5913): 527–530. Bibcode:2009Sci...323..527M. doi:10.1126/science.1166083. PMC 2827536. PMID 19164753.
                    27. ^ Ballara, Angela (1998). Iwi: The Dynamics of Māori Tribal Organisation from c. 1769 to c. 1945 (1st ed.). Wellington: Victoria University Press. ISBN 9780864733283.
                    28. ^ Clark, Ross (1994). "Moriori and Māori: The Linguistic Evidence". In Sutton, Douglas (ed.). The Origins of the First New Zealanders. Auckland: Auckland University Press. pp. 123–135.
                    29. ^ Davis, Denise (September 2007). "The impact of new arrivals". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 April 2010.
                    30. ^ Davis, Denise; Solomon, Māui (March 2009). "'Moriori – The impact of new arrivals'". Te Ara: The Encyclopedia of New Zealand. Retrieved 23 March 2011.
                    31. ^ a b Mein Smith 2005, p. 23.
                    32. ^ Salmond, Anne. Two Worlds: First Meetings Between Maori and Europeans 1642–1772. Auckland: Penguin Books. p. 82. ISBN 0-670-83298-7.
                    33. ^ King 2003, p. 122.
                    34. ^ Fitzpatrick, John (2004). "Food, warfare and the impact of Atlantic capitalism in Aotearo/New Zealand" (PDF). Australasian Political Studies Association Conference: APSA 2004 Conference Papers. Archived from the original (PDF) on 11 May 2011.
                    35. ^ Brailsford, Barry (1972). Arrows of Plague. Wellington: Hick Smith and Sons. p. 35. ISBN 0-456-01060-2.
                    36. ^ Wagstrom, Thor (2005). "Broken Tongues and Foreign Hearts". In Brock, Peggy (ed.). Indigenous Peoples and Religious Change. Boston: Brill Academic Publishers. pp. 71 and 73. ISBN 978-90-04-13899-5.
                    37. ^ Lange, Raeburn (1999). May the people live: a history of Māori health development 1900–1920. Auckland University Press. p. 18. ISBN 978-1-86940-214-3.
                    38. ^ "A Nation sub-divided". Australian Heritage. Heritage Australia Publishing. 2011. Archived from the original on 28 February 2015. Retrieved 27 December 2014.
                    39. ^ a b Rutherford, James (April 2009) [originally published in 1966]. McLintock, Alexander (ed.). Busby, James. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    40. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Sir George Gipps. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    41. ^ a b Wilson, John (March 2009). "Government and nation – The origins of nationhood". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    42. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Settlement from 1840 to 1852. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    43. ^ Foster, Bernard (April 2009) [originally published in 1966]. McLintock, Alexander (ed.). Akaroa, French Settlement At. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    44. ^ Simpson, K (September 2010). "Hobson, William – Biography". In McLintock, Alexander (ed.). Dictionary of New Zealand Biography. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    45. ^ Phillips, Jock (April 2010). "British immigration and the New Zealand Company". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    46. ^ "Crown colony era – the Governor-General". Ministry for Culture and Heritage. March 2009. Retrieved 7 January 2011.
                    47. ^ "New Zealand's 19th-century wars – overview". Ministry for Culture and Heritage. April 2009. Retrieved 7 January 2011.
                    48. ^ a b c d Wilson, John (March 2009). "Government and nation – The constitution". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 February 2011. See pages 2 and 3.
                    49. ^ Temple, Philip (1980). Wellington Yesterday. John McIndoe. ISBN 0-86868-012-5.
                    50. ^ "Parliament moves to Wellington". Ministry for Culture and Heritage. January 2017. Retrieved 27 April 2017.
                    51. ^ a b Wilson, John (March 2009). "History – Liberal to Labour". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 April 2017.
                    52. ^ Hamer, David. "Seddon, Richard John". Dictionary of New Zealand Biography. Ministry for Culture and Heritage. Retrieved 27 April 2017.
                    53. ^ Boxall, Peter; Haynes, Peter (1997). "Strategy and Trade Union Effectiveness in a Neo-liberal Environment". British Journal of Industrial Relations. 35 (4): 567–591. doi:10.1111/1467-8543.00069. Archived from the original (PDF) on 11 May 2011.
                    54. ^ "Proclamation". The London Gazette. No. 28058. 10 September 1907. p. 6149.
                    55. ^ "Dominion status – Becoming a dominion". Ministry for Culture and Heritage. September 2014. Retrieved 26 April 2017.
                    56. ^ "War and Society". Ministry for Culture and Heritage. Retrieved 7 January 2011.
                    57. ^ Easton, Brian (April 2010). "Economic history – Interwar years and the great depression". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    58. ^ Derby, Mark (May 2010). "Strikes and labour disputes – Wars, depression and first Labour government". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    59. ^ Easton, Brian (November 2010). "Economic history – Great boom, 1935–1966". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    60. ^ Keane, Basil (November 2010). "Te Māori i te ohanga – Māori in the economy – Urbanisation". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 January 2011.
                    61. ^ Royal, Te Ahukaramū (March 2009). "Māori – Urbanisation and renaissance". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    62. ^ Healing the past, building a future: A Guide to Treaty of Waitangi Claims and Negotiations with the Crown (PDF). Office of Treaty Settlements. March 2015. ISBN 978-0-478-32436-5. Retrieved 26 April 2017.
                    63. ^ Report on the Crown's Foreshore and Seabed Policy (Report). Ministry of Justice. Retrieved 26 April 2017.
                    64. ^ Barker, Fiona (June 2012). "Debate about the foreshore and seabed". Te Ara: The Encyclopedia of New Zealand. Retrieved 26 April 2017.
                    65. ^ a b "New Zealand's Constitution". The Governor-General of New Zealand. Retrieved 13 January 2010.
                    66. ^ a b c "Factsheet – New Zealand – Political Forces". The Economist. The Economist Group. 15 February 2005. Archived from the original on 14 May 2006. Retrieved 4 August 2009.
                    67. ^ "Royal Titles Act 1974". New Zealand Parliamentary Counsel Office. February 1974. Section 1. Retrieved 8 January 2011.
                    68. ^ "Constitution Act 1986". New Zealand Parliamentary Counsel Office. 1 January 1987. Section 2.1. Retrieved 15 July 2018. The Sovereign in right of New Zealand is the head of State of New Zealand, and shall be known by the royal style and titles proclaimed from time to time. Cite journal requires |journal= (help)
                    69. ^ "The Role of the Governor-General". The Governor-General of New Zealand. Retrieved 6 July 2017.
                    70. ^ Harris, Bruce (2009). "Replacement of the Royal Prerogative in New Zealand". New Zealand Universities Law Review. 23: 285–314. Archived from the original on 18 July 2011. Retrieved 28 August 2016.
                    71. ^ a b "The Reserve Powers". The Governor-General of New Zealand. Retrieved 8 January 2011.
                    72. ^ a b c d "Parliament Brief: What is Parliament?". New Zealand Parliament. Retrieved 30 November 2016.
                    73. ^ McLean, Gavin (February 2015). "Premiers and prime ministers". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 November 2016.
                    74. ^ Wilson, John (November 2010). "Government and nation – System of government". Te Ara: The Encyclopedia of New Zealand. Retrieved 9 January 2011.
                    75. ^ "Principles of Cabinet decision making". Cabinet Manual. Department of the Prime Minister and Cabinet. 2008. Retrieved 1 December 2016.
                    76. ^ "The electoral cycle". Cabinet Manual. Department of the Prime Minister and Cabinet. 2008. Retrieved 30 April 2017.
                    77. ^ a b "First past the post – the road to MMP". Ministry for Culture and Heritage. September 2009. Retrieved 9 January 2011.
                    78. ^ "Reviewing electorate numbers and boundaries". Electoral Commission. 8 May 2005. Archived from the original on 9 November 2011. Retrieved 7 July 2018.
                    79. ^ "Sainte-Laguë allocation formula". Electoral Commission. 4 February 2013. Retrieved 31 May 2014.
                    80. ^ Paxton, Pamela; Hughes, Melanie M. (2015). Women, Politics, and Power: A Global Perspective. CQ Press. p. 107. ISBN 978-1-48-337701-8. Retrieved 25 July 2017.
                    81. ^ "Jacinda Ardern sworn in as new Prime Minister". New Zealand Herald. 26 October 2017. Retrieved 26 October 2017.
                    82. ^ "Female political leaders have been smashing glass ceilings for ages". Stuff.co.nz. Fairfax NZ. 27 October 2017. Retrieved 19 July 2018.
                    83. ^ "Role of the Chief Justice". Courts of New Zealand. Retrieved 9 June 2018.
                    84. ^ "Structure of the court system". Courts of New Zealand. Retrieved 9 June 2018.
                    85. ^ "The Judiciary". Ministry of Justice. Archived from the original on 24 November 2010. Retrieved 9 January 2011.
                    86. ^ "The Fragile States Index 2016". The Fund for Peace. Archived from the original on 4 February 2017. Retrieved 30 November 2016.
                    87. ^ "Democracy Index 2017" (PDF). Economist Intelligence Unit. 2018. p. 5. Retrieved 9 December 2018.
                    88. ^ "Corruption Perceptions Index 2017". Transparency International. 21 February 2018. Retrieved 9 December 2018.
                    89. ^ "New Zealand". Country Reports on Human Rights Practices for 2017. United States Department of State. Retrieved 9 December 2018.
                    90. ^ "New Zealand". OECD Better Life Index. 2016. Retrieved 30 November 2016.
                    91. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. External Relations. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    92. ^ "Michael Joseph Savage". Ministry for Culture and Heritage. July 2010. Retrieved 29 January 2011.
                    93. ^ Patman, Robert (2005). "Globalisation, Sovereignty, and the Transformation of New Zealand Foreign Policy" (PDF). Working Paper 21/05. Centre for Strategic Studies, Victoria University of Wellington. p. 8. Archived from the original (PDF) on 25 September 2007. Retrieved 12 March 2007.
                    94. ^ "Department Of External Affairs: Security Treaty between Australia, New Zealand and the United States of America". Australian Government. September 1951. Archived from the original on 29 June 2011. Retrieved 11 January 2011.
                    95. ^ "The Vietnam War". New Zealand History. Ministry for Culture and Heritage. June 2008. Retrieved 11 January 2011.
                    96. ^ "Sinking the Rainbow Warrior – nuclear-free New Zealand". New Zealand History. Ministry for Culture and Heritage. August 2008. Retrieved 11 January 2011.
                    97. ^ "Nuclear-free legislation – nuclear-free New Zealand". New Zealand History. Ministry for Culture and Heritage. August 2008. Retrieved 11 January 2011.
                    98. ^ Lange, David (1990). Nuclear Free: The New Zealand Way. New Zealand: Penguin Books. ISBN 0-14-014519-2.
                    99. ^ "Australia in brief". Australian Department of Foreign Affairs and Trade. Archived from the original on 22 December 2010. Retrieved 11 January 2011.
                    100. ^ a b "New Zealand country brief". Australian Department of Foreign Affairs and Trade. Retrieved 11 January 2011.
                    101. ^ Collett, John (4 September 2013). "Kiwis face hurdles in pursuit of lost funds". Retrieved 4 October 2013.
                    102. ^ Bertram, Geoff (April 2010). "South Pacific economic relations – Aid, remittances and tourism". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 January 2011.
                    103. ^ Howes, Stephen (November 2010). "Making migration work: Lessons from New Zealand". Development Policy Centre. Retrieved 23 March 2011.
                    104. ^ Ayele, Yoseph (28 September 2017). "The Growing Momentum for Global Impact in New Zealand". Edmund Hillary Fellowship. Retrieved 9 July 2019.
                    105. ^ "Member States of the United Nations". United Nations. Retrieved 11 January 2011.
                    106. ^ "New Zealand". The Commonwealth. Retrieved 1 December 2016.
                    107. ^ "Members and partners". Organisation for Economic Co-operation and Development. Retrieved 11 January 2011.
                    108. ^ "The future of the Five Power Defence Arrangements". The Strategist. 8 November 2012. Retrieved 1 December 2016.
                    109. ^ "About Us: Role and Responsibilities". New Zealand Defence Force. Retrieved 11 January 2011.
                    110. ^ Ayson, Robert (2007). "New Zealand Defence and Security Policy,1990–2005". In Alley, Roderic (ed.). New Zealand In World Affairs, Volume IV: 1990–2005. Wellington: Victoria University Press. p. 132. ISBN 978-0-86473-548-5.
                    111. ^ "The Battle for Crete". New Zealand History. Ministry for Culture and Heritage. May 2010. Retrieved 9 January 2011.
                    112. ^ "El Alamein – The North African Campaign". New Zealand History. Ministry for Culture and Heritage. May 2009. Retrieved 9 January 2011.
                    113. ^ Holmes, Richard (September 2010). "World War Two: The Battle of Monte Cassino". Retrieved 9 January 2011.
                    114. ^ "Gallipoli stirred new sense of national identity says Clark". New Zealand Herald. April 2005. Retrieved 9 January 2011.
                    115. ^ Prideaux, Bruce (2007). Ryan, Chris (ed.). Battlefield tourism: history, place and interpretation. Elsevier Science. p. 18. ISBN 978-0-08-045362-0.
                    116. ^ Burke, Arthur. "The Spirit of ANZAC". ANZAC Day Commemoration Committee. Archived from the original on 26 December 2010. Retrieved 11 January 2011.
                    117. ^ "South African War 1899–1902". Ministry for Culture and Heritage. February 2009. Retrieved 11 January 2011.
                    118. ^ "New Zealand in the Korean War". New Zealand History. Ministry for Culture and Heritage. Retrieved 1 December 2016.
                    119. ^ "NZ and the Malayan Emergency". Ministry for Culture and Heritage. August 2010. Retrieved 11 January 2011.
                    120. ^ "New Zealand Defence Force Overseas Operations". New Zealand Defence Force. January 2008. Archived from the original on 25 January 2008. Retrieved 17 February 2008.
                    121. ^ a b "New Zealand's Nine Provinces (1853–76)" (PDF). Friends of the Hocken Collections. University of Otago. March 2000. Retrieved 13 January 2011.
                    122. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Provincial Divergencies. An Encyclopaedia of New Zealand. Retrieved 7 January 2011.
                    123. ^ Swarbrick, Nancy (September 2016). "Public holidays". Te Ara: The Encyclopedia of New Zealand. Retrieved 25 June 2017.
                    124. ^ "Overview – regional rugby". Ministry for Culture and Heritage. September 2010. Retrieved 13 January 2011.
                    125. ^ Dollery, Brian; Keogh, Ciaran; Crase, Lin (2007). "Alternatives to Amalgamation in Australian Local Government: Lessons from the New Zealand Experience" (PDF). Sustaining Regions. 6 (1): 50–69. Archived from the original (PDF) on 29 August 2007.
                    126. ^ a b c Sancton, Andrew (2000). Merger mania: the assault on local government. McGill-Queen's University Press. p. 84. ISBN 0-7735-2163-1.
                    127. ^ "Subnational population estimates at 30 June 2010 (boundaries at 1 November 2010)". Statistics New Zealand. 26 October 2010. Archived from the original on 10 June 2011. Retrieved 2 April 2011.
                    128. ^ Smelt & Jui Lin 2009, p. 33.
                    129. ^ a b "Glossary". Department of Internal Affairs. Retrieved 28 August 2016.
                    130. ^ "Chatham Islands Council Act 1995 No 41". New Zealand Parliamentary Counsel Office. 29 July 1995. Retrieved 8 August 2017.
                    131. ^ Gimpel, Diane (2011). Monarchies. ABDO Publishing Company. p. 22. ISBN 978-1-617-14792-0. Retrieved 18 November 2016.
                    132. ^ "System of Government". Government of Niue. Archived from the original on 13 November 2010. Retrieved 13 January 2010.
                    133. ^ "Government – Structure, Personnel". Government of the Cook Islands. Retrieved 13 January 2010.
                    134. ^ "Tokelau Government". Government of Tokelau. Retrieved 16 November 2016.
                    135. ^ "Scott Base". Antarctica New Zealand. Retrieved 13 January 2010.
                    136. ^ "Citizenship Act 1977 No 61". Zealand Parliamentary Counsel Office. 1 December 1977. Retrieved 26 May 2017.
                    137. ^ "Check if you're a New Zealand citizen". Department of Internal Affairs. Retrieved 20 January 2015.
                    138. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. The Sea Floor. An Encyclopaedia of New Zealand. Retrieved 13 January 2011.
                    139. ^ "Hauraki Gulf islands". Auckland City Council. Archived from the original on 25 December 2010. Retrieved 13 January 2011.
                    140. ^ Hindmarsh (2006). "Discovering D'Urville". Heritage New Zealand. Archived from the original on 11 May 2011. Retrieved 13 January 2011.
                    141. ^ "Distance tables". Auckland Coastguard. Archived from the original on 23 January 2011. Retrieved 2 March 2011.
                    142. ^ McKenzie, D. W. (1987). Heinemann New Zealand atlas. Heinemann Publishers. ISBN 0-7900-0187-X.
                    143. ^ a b c d e f "CIA – The World Factbook". Cia.gov. Retrieved 4 May 2013.
                    144. ^ "Geography". Statistics New Zealand. 1999. Archived from the original on 22 May 2010. Retrieved 21 December 2009.
                    145. ^ Offshore Options: Managing Environmental Effects in New Zealand's Exclusive Economic Zone (PDF). Wellington: Ministry for the Environment. 2005. ISBN 0-478-25916-6. Retrieved 23 June 2017.
                    146. ^ Coates, Glen (2002). The rise and fall of the Southern Alps. Canterbury University Press. p. 15. ISBN 0-908812-93-0.
                    147. ^ Garden 2005, p. 52.
                    148. ^ Grant, David (March 2009). "Southland places – Fiordland's coast". Te Ara: The Encyclopedia of New Zealand. Retrieved 14 January 2011.
                    149. ^ "Central North Island volcanoes". Department of Conservation. Archived from the original on 29 December 2010. Retrieved 14 January 2011.
                    150. ^ Walrond, Carl (March 2009). "Natural environment – Geography and geology". Te Ara: The Encyclopedia of New Zealand. Retrieved 14 January 2010.
                    151. ^ "Taupo". GNS Science. Archived from the original on 24 March 2011. Retrieved 2 April 2011.
                    152. ^ a b Lewis, Keith; Nodder, Scott; Carter, Lionel (March 2009). "Sea floor geology – Active plate boundaries". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    153. ^ Wallis, G. P.; Trewick, S. A. (2009). "New Zealand phylogeography: evolution on a small continent". Molecular Ecology. 18 (17): 3548–3580. doi:10.1111/j.1365-294X.2009.04294.x. PMID 19674312.
                    154. ^ Wright, Dawn; Bloomer, Sherman; MacLeod, Christopher; Taylor, Brian; Goodliffe, Andrew (2000). "Bathymetry of the Tonga Trench and Forearc: A Map Series". Marine Geophysical Researches. 21 (5): 489–512. Bibcode:2000MarGR..21..489W. doi:10.1023/A:1026514914220.
                    155. ^ "Australasia". New Zealand Oxford Dictionary. Oxford University Press. 2005. doi:10.1093/acref/9780195584516.001.0001. ISBN 9780195584516.
                    156. ^ Hobbs, Joseph J. (2016). Fundamentals of World Regional Geography. Cengage Learning. p. 367. ISBN 9781305854956.
                    157. ^ Hillstrom, Kevin; Hillstrom, Laurie Collier (2003). Australia, Oceania, and Antarctica: A Continental Overview of Environmental Issues. 3. ABC-CLIO. p. 25. ISBN 9781576076941. …defined here as the continent nation of Australia, New Zealand, and twenty-two other island countries and territories sprinkled over more than 40 million square kilometres of the South Pacific.
                    158. ^ a b Mullan, Brett; Tait, Andrew; Thompson, Craig (March 2009). "Climate – New Zealand's climate". Te Ara: The Encyclopedia of New Zealand. Retrieved 15 January 2011.
                    159. ^ "Summary of New Zealand climate extremes". National Institute of Water and Atmospheric Research. 2004. Retrieved 30 April 2010.
                    160. ^ Walrond, Carl (March 2009). "Natural environment – Climate". Te Ara: The Encyclopedia of New Zealand. Retrieved 15 January 2011.
                    161. ^ "Mean monthly rainfall". National Institute of Water and Atmospheric Research. Archived from the original (XLS) on 3 May 2011. Retrieved 4 February 2011.
                    162. ^ "Mean monthly sunshine hours". National Institute of Water and Atmospheric Research. Archived from the original (XLS) on 15 October 2008. Retrieved 4 February 2011.
                    163. ^ "New Zealand climate and weather". Tourism New Zealand. Retrieved 13 November 2016.
                    164. ^ "Climate data and activities". National Institute of Water and Atmospheric Research. Retrieved 11 February 2016.
                    165. ^ Cooper, R.; Millener, P. (1993). "The New Zealand biota: Historical background and new research". Trends in Ecology & Evolution. 8 (12): 429. doi:10.1016/0169-5347(93)90004-9.
                    166. ^ Trewick SA, Morgan-Richards M. 2014. New Zealand Wild Life. Penguin, New Zealand. ISBN 9780143568896
                    167. ^ Lindsey, Terence; Morris, Rod (2000). Collins Field Guide to New Zealand Wildlife. HarperCollins (New Zealand) Limited. p. 14. ISBN 978-1-86950-300-0.
                    168. ^ a b "Frequently asked questions about New Zealand plants". New Zealand Plant Conservation Network. May 2010. Retrieved 15 January 2011.
                    169. ^ De Lange, Peter James; Sawyer, John William David & Rolfe, Jeremy (2006). New Zealand indigenous vascular plant checklist. New Zealand Plant Conservation Network. ISBN 0-473-11306-6.
                    170. ^ Wassilieff, Maggy (March 2009). "Lichens – Lichens in New Zealand". Te Ara: The Encyclopedia of New Zealand. Retrieved 16 January 2011.
                    171. ^ McLintock, Alexander, ed. (April 2010) [originally published in 1966]. Mixed Broadleaf Podocarp and Kauri Forest. An Encyclopaedia of New Zealand. Retrieved 15 January 2011.
                    172. ^ Mark, Alan (March 2009). "Grasslands – Tussock grasslands". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 January 2010.
                    173. ^ "Commentary on Forest Policy in the Asia-Pacific Region (A Review for Indonesia, Malaysia, New Zealand, Papua New Guinea, Philippines, Thailand and Western Samoa)". Forestry Department. 1997. Retrieved 4 February 2011.
                    174. ^ McGlone, M.S. (1989). "The Polynesian settlement of New Zealand in relation to environmental and biotic changes" (PDF). New Zealand Journal of Ecology. 12(S): 115–129. Archived from the original (PDF) on 17 July 2014.
                    175. ^ Taylor, R. and Smith, I. (1997). The state of New Zealand’s environment 1997. Ministry for the Environment, Wellington.
                    176. ^ "New Zealand ecology: Flightless birds". TerraNature. Retrieved 17 January 2011.
                    177. ^ a b Holdaway, Richard (March 2009). "Extinctions – New Zealand extinctions since human arrival". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    178. ^ Kirby, Alex (January 2005). "Huge eagles 'dominated NZ skies'". BBC News. Retrieved 4 February 2011.
                    179. ^ "Reptiles and frogs". Department of Conservation. Archived from the original on 29 January 2015. Retrieved 25 June 2017.
                    180. ^ Pollard, Simon (September 2007). "Spiders and other arachnids". Te Ara: The Encyclopedia of New Zealand. Retrieved 25 June 2017.
                    181. ^ "Wētā". Department of Conservation. Retrieved 25 June 2017.
                    182. ^ Ryan, Paddy (March 2009). "Snails and slugs – Flax snails, giant snails and veined slugs". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    183. ^ Herrera-Flores, Jorge A.; Stubbs, Thomas L.; Benton, Michael J.; Ruta, Marcello (May 2017). "Macroevolutionary patterns in Rhynchocephalia: is the tuatara (Sphenodon punctatus) a living fossil?". Palaeontology. 60 (3): 319–328. doi:10.1111/pala.12284.
                    184. ^ "Tiny Bones Rewrite Textbooks, first New Zealand land mammal fossil". University of New South Wales. 31 May 2007. Archived from the original on 31 May 2007.
                    185. ^ Worthy, Trevor H.; Tennyson, Alan J. D.; Archer, Michael; Musser, Anne M.; Hand, Suzanne J.; Jones, Craig; Douglas, Barry J.; McNamara, James A.; Beck, Robin M. D. (2006). "Miocene mammal reveals a Mesozoic ghost lineage on insular New Zealand, southwest Pacific". Proceedings of the National Academy of Sciences of the United States of America. 103 (51): 19419–23. Bibcode:2006PNAS..10319419W. doi:10.1073/pnas.0605684103. PMC 1697831. PMID 17159151.
                    186. ^ "Marine Mammals". Department of Conservation. Archived from the original on 8 March 2011. Retrieved 17 January 2011.
                    187. ^ "Sea and shore birds". Department of Conservation. Retrieved 7 March 2011.
                    188. ^ "Penguins". Department of Conservation. Retrieved 7 March 2011.
                    189. ^ Jones, Carl (2002). "Reptiles and Amphibians". In Perrow, Martin; Davy, Anthony (eds.). Handbook of ecological restoration: Principles of Restoration. 2. Cambridge University Press. p. 362. ISBN 0-521-79128-6.
                    190. ^ Towns, D.; Ballantine, W. (1993). "Conservation and restoration of New Zealand Island ecosystems". Trends in Ecology & Evolution. 8 (12): 452. doi:10.1016/0169-5347(93)90009-E.
                    191. ^ Rauzon, Mark (2008). "Island restoration: Exploring the past, anticipating the future" (PDF). Marine Ornithology. 35: 97–107.
                    192. ^ Diamond, Jared (1990). Towns, D; Daugherty, C; Atkinson, I (eds.). New Zealand as an archipelago: An international perspective (PDF). Wellington: Conservation Sciences Publication No. 2. Department of Conservation. pp. 3–8.
                    193. ^ World Economic Outlook. International Monetary Fund. April 2018. p. 63. ISBN 978-1-48434-971-7. Retrieved 21 June 2018.
                    194. ^ "Rankings on Economic Freedom". The Heritage Foundation. 2016. Retrieved 30 November 2016.
                    195. ^ "Currencies of the territories listed in the BS exchange rate lists". Bank of Slovenia. Retrieved 22 January 2011.
                    196. ^ a b c McLintock, Alexander, ed. (November 2009) [originally published in 1966]. Historical evolution and trade patterns. An Encyclopaedia of New Zealand. Retrieved 10 February 2011.
                    197. ^ Stringleman, Hugh; Peden, Robert (October 2009). "Sheep farming – Growth of the frozen meat trade, 1882–2001". Te Ara: The Encyclopedia of New Zealand. Retrieved 6 May 2010.
                    198. ^ Baker, John (February 2010) [1966]. McLintock, Alexander (ed.). Some Indicators of Comparative Living Standards. An Encyclopaedia of New Zealand. Retrieved 30 April 2010. PDF Table
                    199. ^ Wilson, John (March 2009). "History – The later 20th century". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 February 2011.
                    200. ^ Nixon, Chris; Yeabsley, John (April 2010). "Overseas trade policy – Difficult times – the 1970s and early 1980s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    201. ^ Evans, N. "Up From Down Under: After a Century of Socialism, Australia and New Zealand are Cutting Back Government and Freeing Their Economies". National Review. 46 (16): 47–51.
                    202. ^ Trade, Food Security, and Human Rights: The Rules for International Trade in Agricultural Products and the Evolving World Food Crisis. Routledge. 2016. p. 125. ISBN 9781317008521.
                    203. ^ Wayne Arnold (2 August 2007). "Surviving Without Subsidies". The New York Times. Retrieved 11 August 2015. ... ever since a liberal but free-market government swept to power in 1984 and essentially canceled handouts to farmers ... They went cold turkey and in the process it was very rough on their farming economy
                    204. ^ Easton, Brian (November 2010). "Economic history – Government and market liberalisation". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 February 2011.
                    205. ^ Hazledine, Tim (1998). Taking New Zealand Seriously: The Economics of Decency (PDF). HarperCollins Publishers. ISBN 1-86950-283-3. Archived from the original (PDF) on 10 May 2011.
                    206. ^ "NZ tops Travellers' Choice Awards". Stuff Travel. May 2008. Retrieved 30 April 2010.
                    207. ^ a b c "Unemployment: the Social Report 2016 – Te pūrongo oranga tangata". Ministry of Social Development. Retrieved 18 August 2017.
                    208. ^ "New Zealand Takes a Pause in Cutting Rates". The New York Times. 10 June 2009. Retrieved 30 April 2010.
                    209. ^ "New Zealand's slump longest ever". BBC News. 26 June 2009. Retrieved 30 April 2010.
                    210. ^ Bascand, Geoff (February 2011). "Household Labour Force Survey: December 2010 quarter – Media Release". Statistics New Zealand. Archived from the original on 29 April 2011. Retrieved 4 February 2011.
                    211. ^ Davenport, Sally (2004). "Panic and panacea: brain drain and science and technology human capital policy". Research Policy. 33 (4): 617–630. doi:10.1016/j.respol.2004.01.006.
                    212. ^ O'Hare, Sean (September 2010). "New Zealand brain-drain worst in world". The Daily Telegraph. United Kingdom.
                    213. ^ Collins, Simon (March 2005). "Quarter of NZ's brightest are gone". New Zealand Herald.
                    214. ^ Winkelmann, Rainer (2000). "The labour market performance of European immigrants in New Zealand in the 1980s and 1990s". The International Migration Review. The Center for Migration Studies of New York. 33 (1): 33–58. doi:10.2307/2676011. JSTOR 2676011. Journal subscription required
                    215. ^ Bain 2006, p. 44.
                    216. ^ "GII 2016 Report" (PDF). Global Innovation Index. Retrieved 21 June 2018.
                    217. ^ Groser, Tim (March 2009). "Speech to ASEAN-Australia-New Zealand Free Trade Agreement Seminars". New Zealand Government. Retrieved 30 January 2011.
                    218. ^ "Improving Access to Markets:Agriculture". New Zealand Ministry of Foreign Affairs and Trade. Retrieved 22 January 2011.
                    219. ^ "Standard International Trade Classification R4 – Exports (Annual-Jun)". Statistics New Zealand. April 2015. Retrieved 3 April 2015.
                    220. ^ a b c "Goods and services trade by country: Year ended June 2018 – corrected". Statistics New Zealand. Retrieved 17 February 2019.
                    221. ^ "China and New Zealand sign free trade deal". The New York Times. April 2008.
                    222. ^ a b "Key Tourism Statistics" (PDF). Ministry of Business, Innovation and Employment. 26 April 2017. Retrieved 26 April 2017.
                    223. ^ Easton, Brian (March 2009). "Economy – Agricultural production". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    224. ^ Stringleman, Hugh; Peden, Robert (March 2009). "Sheep farming – Changes from the 20th century". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    225. ^ Stringleman, Hugh; Scrimgeour, Frank (November 2009). "Dairying and dairy products – Dairying in the 2000s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    226. ^ Stringleman, Hugh; Scrimgeour, Frank (March 2009). "Dairying and dairy products – Dairy exports". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    227. ^ Stringleman, Hugh; Scrimgeour, Frank (March 2009). "Dairying and dairy products – Manufacturing and marketing in the 2000s". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    228. ^ Dalley, Bronwyn (March 2009). "Wine – The wine boom, 1980s and beyond". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    229. ^ "Wine in New Zealand". The Economist. 27 March 2008. Retrieved 29 April 2017.
                    230. ^ "Agricultural and forestry exports from New Zealand: Primary sector export values for the year ending June 2010". Ministry of Agriculture and Forestry. 14 January 2011. Archived from the original on 10 May 2011. Retrieved 8 April 2011.
                    231. ^ a b Energy in New Zealand 2016 (PDF) (Report). Ministry of Business, Innovation and Employment. September 2016. p. 47. ISSN 2324-5913. Archived from the original (PDF) on 3 May 2017.
                    232. ^ "Appendix 1: Technical information about drinking water supply in the eight local authorities". Office of the Auditor-General. Retrieved 2 September 2016.
                    233. ^ "Water supply". Greater Wellington Regional Council. Retrieved 2 September 2016.
                    234. ^ "State highway frequently asked questions". NZ Transport Agency. Retrieved 28 April 2017.
                    235. ^ Humphris, Adrian (April 2010). "Public transport – Passenger trends". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    236. ^ Atkinson, Neill (November 2010). "Railways – Rail transformed". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    237. ^ "About Metlink". Retrieved 27 December 2016.
                    238. ^ Atkinson, Neill (April 2010). "Railways – Freight transport". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    239. ^ "International Visitors" (PDF). Ministry of Economic Development. June 2009. Retrieved 30 January 2011.
                    240. ^ "10. Airports". Infrastructure Stocktake: Infrastructure Audit. Ministry of Economic Development. December 2005. Archived from the original on 22 May 2010. Retrieved 30 January 2011.
                    241. ^ a b Wilson, A. C. (March 2010). "Telecommunications - Telecom". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 August 2017.
                    242. ^ "Telecom separation". Ministry of Business, Innovation and Employment. 14 September 2015. Retrieved 11 August 2017.
                    243. ^ "Broadband and mobile programmes - Ministry of Business, Innovation & Employment". www.mbie.govt.nz.
                    244. ^ "2017 Global ICT Development Index". International Telecommunication Union (ITU). 2018. Retrieved 18 September 2018.
                    245. ^ "2013 Census usually resident population counts". Statistics New Zealand. 14 October 2013. Retrieved 20 September 2018.
                    246. ^ "National population projections: 2016(base)–2068" (Press release). Statistics New Zealand. 18 October 2016. Retrieved 20 October 2018.
                    247. ^ "Subnational population estimates at 30 June 2009". Statistics New Zealand. 30 June 2007. Retrieved 30 April 2010.
                    248. ^ "Quality of Living Ranking 2016". London: Mercer. 23 February 2016. Retrieved 28 April 2017.
                    249. ^ "NZ life expectancy among world's best". Stuff.co.nz. Fairfax NZ. Retrieved 6 July 2014.
                    250. ^ a b Department of Economic and Social Affairs Population Division (2009). "World Population Prospects" (PDF). 2008 revision. United Nations. Retrieved 29 August 2009. Cite journal requires |journal= (help)
                    251. ^ "World Factbook EUROPE : NEW ZEALAND", The World Factbook, 12 July 2018
                    252. ^ "New Zealand mortality statistics: 1950 to 2010" (PDF). Ministry of Health of New Zealand. 2 March 2011. Retrieved 16 November 2016.
                    253. ^ "Health expenditure and financing". stats.oecd.org. OECD. 2016. Retrieved 8 December 2017.
                    254. ^ "Subnational Population Estimates: At 30 June 2018 (provisional)". Statistics New Zealand. 23 October 2018. Retrieved 23 October 2018. For urban areas, "Subnational population estimates (UA, AU), by age and sex, at 30 June 1996, 2001, 2006-18 (2017 boundaries)". Statistics New Zealand. 23 October 2018. Retrieved 23 October 2018.
                    255. ^ "2013 Census QuickStats about culture and identity – Ethnic groups in New Zealand". Statistics New Zealand. Retrieved 29 August 2014.
                    256. ^ Pool, Ian (May 2011). "Population change - Key population trends". Te Ara: The Encyclopedia of New Zealand. Archived from the original on 18 August 2017. Retrieved 18 August 2017.
                    257. ^ Dalby, Simon (September 1993). "The 'Kiwi disease': geopolitical discourse in Aotearoa/New Zealand and the South Pacific". Political Geography. 12 (5): 437–456. doi:10.1016/0962-6298(93)90012-V.
                    258. ^ Callister, Paul (2004). "Seeking an Ethnic Identity: Is "New Zealander" a Valid Ethnic Category?" (PDF). New Zealand Population Review. 30 (1&2): 5–22.
                    259. ^ Bain 2006, p. 31.
                    260. ^ "Draft Report of a Review of the Official Ethnicity Statistical Standard: Proposals to Address the 'New Zealander' Response Issue" (PDF). Statistics New Zealand. April 2009. Retrieved 18 January 2011.
                    261. ^ Ranford, Jodie. "'Pakeha', Its Origin and Meaning". Māori News. Retrieved 20 February 2008. Originally the Pakeha were the early European settlers, however, today ‘Pakeha’ is used to describe any peoples of non-Maori or non-Polynesian heritage. Pakeha is not an ethnicity but rather a way to differentiate between the historical origins of our settlers, the Polynesians and the Europeans, the Maori and the other
                    262. ^ Socidad Peruana de Medicina Intensiva (SOPEMI) (2000). Trends in international migration: continuous reporting system on migration. Organisation for Economic Co-operation and Development. pp. 276–278.
                    263. ^ Walrond, Carl (21 September 2007). "Dalmatians". Te Ara: The Encyclopedia of New Zealand. Retrieved 30 April 2010.
                    264. ^ "Peoples". Te Ara: The Encyclopedia of New Zealand. 2005. Retrieved 2 June 2017.
                    265. ^ a b Phillips, Jock (11 August 2015). "History of immigration". Te Ara: The Encyclopedia of New Zealand. Retrieved 2 June 2017.
                    266. ^ Brawley, Sean (1993). "'No White Policy in NZ': Fact and Fiction in New Zealand's Asian Immigration Record, 1946-1978" (PDF). New Zealand Journal of History. 27 (1): 33–36. Retrieved 2 June 2017.
                    267. ^ "International Migration Outlook – New Zealand 2009/10" (PDF). New Zealand Department of Labour. 2010. p. 2. ISSN 1179-5085. Archived from the original (PDF) on 11 May 2011. Retrieved 16 April 2011.
                    268. ^ "2013 Census QuickStats about culture and identity – Birthplace and people born overseas". Statistics New Zealand. Retrieved 29 August 2014.
                    269. ^ Butcher, Andrew; McGrath, Terry (2004). "International Students in New Zealand: Needs and Responses" (PDF). International Education Journal. 5 (4).
                    270. ^ 2013 Census QuickStats, Statistics New Zealand, 2013, ISBN 978-0-478-40864-5
                    271. ^ a b c "2013 Census QuickStats about culture and identity – Languages spoken". Statistics New Zealand. Retrieved 8 September 2016.
                    272. ^ Hay, Maclagan & Gordon 2008, p. 14.
                    273. ^ * Bauer, Laurie; Warren, Paul; Bardsley, Dianne; Kennedy, Marianna; Major, George (2007), "New Zealand English", Journal of the International Phonetic Association, 37 (1): 97–102, doi:10.1017/S0025100306002830
                    274. ^ a b Phillips, Jock (March 2009). "The New Zealanders – Bicultural New Zealand". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    275. ^ Squires, Nick (May 2005). "British influence ebbs as New Zealand takes to talking Māori". The Daily Telegraph. Retrieved 3 May 2017.
                    276. ^ "Waitangi Tribunal claim – Māori Language Week". Ministry for Culture and Heritage. July 2010. Retrieved 19 January 2011.
                    277. ^ "Ngā puna kōrero: Where Māori speak te reo – infographic". Statistics New Zealand. Retrieved 8 September 2016.
                    278. ^ Drinnan, John (8 July 2016). "'Maori' will remain in the name Maori Television". New Zealand Herald. Retrieved 28 August 2016. According to 2015 figures supplied by Maori TV, its two channels broadcast an average of 72 per cent Maori language content - 59 per cent on the main channel and 99 per cent on te reo.
                    279. ^ "Ngāi Tahu Claims Settlement Act 1998". New Zealand Parliamentary Counsel Office. 20 May 2014 [1 October 1998]. Retrieved 10 March 2019.
                    280. ^ New Zealand Sign Language Act 2006 No 18 (as at 30 June 2008), Public Act. New Zealand Parliamentary Counsel Office. Retrieved 29 November 2011.
                    281. ^ Zuckerman, Phil (2006). Martin, Michael (ed.). The Cambridge Companion to Atheism (PDF). Cambridge University Press. pp. 47–66. ISBN 978-0-521-84270-9. Retrieved 8 August 2017.
                    282. ^ Walrond, Carl (May 2012). "Atheism and secularism – Who is secular?". Te Ara: The Encyclopedia of New Zealand. Retrieved 8 August 2017.
                    283. ^ a b "2013 Census QuickStats about culture and identity – Religious affiliation". Statistics New Zealand. 15 April 2014. Retrieved 25 July 2015.
                    284. ^ a b c "2013 Census QuickStats about culture and identity – tables". Statistics New Zealand. 15 April 2014. Retrieved 14 April 2018. Excel download
                    285. ^ Kaa, Hirini (May 2011). "Māori and Christian denominations". Te Ara: The Encyclopedia of New Zealand. Retrieved 20 April 2017.
                    286. ^ Morris, Paul (May 2011). "Diverse religions". Te Ara: The Encyclopedia of New Zealand. Retrieved 20 April 2017.
                    287. ^ a b Dench, Olivia (July 2010). "Education Statistics of New Zealand: 2009". Education Counts. Retrieved 19 January 2011.
                    288. ^ "Education Act 1989 No 80". New Zealand Parliamentary Counsel Office. 1989. Section 3. Retrieved 5 January 2013.
                    289. ^ "Education Act 1989 No 80 (as at 01 February 2011), Public Act. Part 14: Establishment and disestablishment of tertiary institutions, Section 62: Establishment of institutions". Education Act 1989 No 80. New Zealand Parliamentary Counsel Office. 1 February 2011. Retrieved 15 August 2011.
                    290. ^ "Studying in New Zealand: Tertiary education". New Zealand Qualifications Authority. Retrieved 15 August 2011.
                    291. ^ "Educational attainment of the population". Education Counts. 2006. Archived from the original (xls) on 15 October 2008. Retrieved 21 February 2008.
                    292. ^ "What Students Know and Can Do: Student Performance in Reading, Mathematics and Science 2010" (PDF). OECD. Retrieved 21 July 2012.
                    293. ^ Kennedy 2007, p. 398.
                    294. ^ Hearn, Terry (March 2009). "English – Importance and influence". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    295. ^ "Conclusions – British and Irish immigration". Ministry for Culture and Heritage. March 2007. Retrieved 21 January 2011.
                    296. ^ Stenhouse, John (November 2010). "Religion and society – Māori religion". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    297. ^ "Māori Social Structures". Ministry of Justice. March 2001. Retrieved 21 January 2011.
                    298. ^ "Thousands turn out for Pasifika Festival". Radio New Zealand. 25 March 2017. Retrieved 18 August 2017.
                    299. ^ Kennedy 2007, p. 400.
                    300. ^ Kennedy 2007, p. 399.
                    301. ^ Phillips, Jock (March 2009). "The New Zealanders – Post-war New Zealanders". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    302. ^ Phillips, Jock (March 2009). "The New Zealanders – Ordinary blokes and extraordinary sheilas". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    303. ^ Phillips, Jock (March 2009). "Rural mythologies – The cult of the pioneer". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    304. ^ Barker, Fiona (June 2012). "New Zealand identity – Culture and arts". Te Ara: The Encyclopedia of New Zealand. Retrieved 7 December 2016.
                    305. ^ a b Wilson, John (September 2016). "Nation and government – Nationhood and identity". Te Ara: The Encyclopedia of New Zealand. Retrieved 3 December 2016.
                    306. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Visual arts and crafts". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    307. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Elements of Carving. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    308. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Surface Patterns. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    309. ^ McKay, Bill (2004). "Māori architecture: transforming western notions of architecture". Fabrications. 14 (1&2): 1–12. doi:10.1080/10331867.2004.10525189. Archived from the original on 13 May 2011.
                    310. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Painted Designs. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    311. ^ McLintock, Alexander, ed. (2009) [1966]. Tattooing. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    312. ^ a b "Beginnings – history of NZ painting". Ministry for Culture and Heritage. December 2010. Retrieved 17 February 2011.
                    313. ^ "A new New Zealand art – history of NZ painting". Ministry for Culture and Heritage. November 2010. Retrieved 16 February 2011.
                    314. ^ "Contemporary Maori art". Ministry for Culture and Heritage. November 2010. Retrieved 16 February 2011.
                    315. ^ Rauer, Julie. "Paradise Lost: Contemporary Pacific Art At The Asia Society". Asia Society and Museum. Retrieved 17 February 2011.
                    316. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Textile Designs. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    317. ^ Keane, Basil (March 2009). "Pounamu – jade or greenstone – Implements and adornment". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 February 2011.
                    318. ^ Wilson, John (March 2009). "Society – Food, drink and dress". Te Ara: The Encyclopedia of New Zealand. Retrieved 17 February 2011.
                    319. ^ Swarbrick, Nancy (June 2010). "Creative life – Design and fashion". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    320. ^ a b "Fashion in New Zealand – New Zealand's fashion industry". The Economist. 28 February 2008. Retrieved 6 August 2009.
                    321. ^ Swarbrick, Nancy (June 2010). "Creative life – Writing and publishing". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2011.
                    322. ^ "The making of New Zealand literature". Ministry for Culture and Heritage. November 2010. Retrieved 22 January 2011.
                    323. ^ "New directions in the 1930s – New Zealand literature". Ministry for Culture and Heritage. August 2008. Retrieved 12 February 2011.
                    324. ^ "The war and beyond – New Zealand literature". Ministry for Culture and Heritage. November 2007. Retrieved 12 February 2011.
                    325. ^ "28 cities join the UNESCO Creative Cities Network". UNESCO. December 2014. Retrieved 7 March 2015.
                    326. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Music". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    327. ^ McLintock, Alexander, ed. (April 2009) [originally published in 1966]. Maori Music. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    328. ^ McLintock, Alexander, ed. (April 2009) [1966]. Musical Instruments. An Encyclopaedia of New Zealand. Retrieved 16 February 2011.
                    329. ^ McLintock, Alexander, ed. (April 2009) [1966]. Instruments Used for Non-musical Purposes. An Encyclopaedia of New Zealand. Retrieved 16 February 2011.
                    330. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: General History. An Encyclopaedia of New Zealand. Retrieved 15 February 2011.
                    331. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: Brass Bands. An Encyclopaedia of New Zealand. Retrieved 14 April 2011.
                    332. ^ McLintock, Alexander, ed. (April 2009) [1966]. Music: Pipe Bands. An Encyclopaedia of New Zealand. Retrieved 14 April 2011.
                    333. ^ Swarbrick, Nancy (June 2010). "Creative life – Performing arts". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    334. ^ "History – celebrating our music since 1965". Recording Industry Association of New Zealand. 2008. Archived from the original on 14 September 2011. Retrieved 23 January 2012.
                    335. ^ "About RIANZ – Introduction". Recording Industry Association of New Zealand. Archived from the original on 21 December 2011. Retrieved 23 January 2012.
                    336. ^ Downes, Siobhan (1 January 2017). "World famous in New Zealand: Hobbiton Movie Set". Stuff Travel. Retrieved 6 July 2017.
                    337. ^ Brian, Pauling (October 2014). "Radio – The early years, 1921 to 1932". Te Ara: The Encyclopedia of New Zealand. Retrieved 6 July 2017.
                    338. ^ "New Zealand's first official TV broadcast". Ministry for Culture and Heritage. December 2016. Retrieved 6 July 2017.
                    339. ^ a b Swarbrick, Nancy (June 2010). "Creative life – Film and broadcasting". Te Ara: The Encyclopedia of New Zealand. Retrieved 21 January 2011.
                    340. ^ Horrocks, Roger. "A History of Television in New Zealand". NZ On Screen. Retrieved 13 September 2017.
                    341. ^ "Top 10 Highest Grossing New Zealand Movies Ever". Flicks.co.nz. May 2016. Retrieved 11 August 2017.
                    342. ^ Cieply, Michael; Rose, Jeremy (October 2010). "New Zealand Bends and 'Hobbit' Stays". The New York Times. Retrieved 11 August 2017.
                    343. ^ "Production Guide: Locations". Film New Zealand. Archived from the original on 7 November 2010. Retrieved 21 January 2011.
                    344. ^ Myllylahti, Merja (December 2016). JMAD New Zealand Media Ownership Report 2016 (PDF) (Report). Auckland University of Technology. pp. 4–29. Archived from the original (PDF) on 21 May 2017. Retrieved 11 August 2017.
                    345. ^ "Scores and Status Data 1980-2015". Freedom of the Press 2015. Freedom House. Retrieved 23 November 2016.
                    346. ^ Hearn, Terry (March 2009). "English – Popular culture". Te Ara: The Encyclopedia of New Zealand. Retrieved 22 January 2012.
                    347. ^ "Sport, Fitness and Leisure". New Zealand Official Yearbook. Statistics New Zealand. 2000. Archived from the original on 7 June 2011. Retrieved 21 July 2008. Traditionally New Zealanders have excelled in rugby union, which is regarded as the national sport, and track and field athletics.
                    348. ^ a b Phillips, Jock (February 2011). "Sports and leisure – Organised sports". Te Ara: The Encyclopedia of New Zealand. Retrieved 23 March 2011.
                    349. ^ a b "More and more students wear school sports colours". New Zealand Secondary School Sports Council. Retrieved 30 March 2015.
                    350. ^ Crawford, Scott (January 1999). "Rugby and the Forging of National Identity". In Nauright, John (ed.). Sport, Power And Society In New Zealand: Historical And Contemporary Perspectives (PDF). ASSH Studies In Sports History. Archived from the original (PDF) on 19 January 2012. Retrieved 22 January 2011.
                    351. ^ "Rugby, racing and beer". Ministry for Culture and Heritage. August 2010. Retrieved 22 January 2011.
                    352. ^ Derby, Mark (December 2010). "Māori–Pākehā relations – Sports and race". Te Ara: The Encyclopedia of New Zealand. Retrieved 4 February 2011.
                    353. ^ Bain 2006, p. 69.
                    354. ^ Langton, Graham (1996). A history of mountain climbing in New Zealand to 1953 (Thesis). Christchurch: University of Canterbury. p. 28. Retrieved 12 August 2017.
                    355. ^ "World mourns Sir Edmund Hillary". The Age. Melbourne. 11 January 2008.
                    356. ^ "Sport and Recreation Participation Levels" (PDF). Sport and Recreation New Zealand. 2009. Archived from the original (PDF) on 15 January 2015. Retrieved 27 November 2016.
                    357. ^ Barclay-Kerr, Hoturoa (September 2013). "Waka ama – outrigger canoeing". Te Ara: The Encyclopedia of New Zealand. Retrieved 12 August 2017.
                    358. ^ "NZ's first Olympic century". Ministry for Culture and Heritage. August 2016. Retrieved 27 April 2017.
                    359. ^ "London 2012 Olympic Games: Medal strike rate – Final count (revised)". Statistics New Zealand. 14 August 2012. Retrieved 4 December 2013.
                    360. ^ "Rio 2016 Olympic Games: Medals per capita". Statistics New Zealand. 30 August 2016. Retrieved 27 April 2017.
                    361. ^ Kerr, James (14 November 2013). "The All Blacks guide to being successful (off the field)". The Daily Telegraph. London. Retrieved 4 December 2013.
                    362. ^ Fordyce, Tom (23 October 2011). "2011 Rugby World Cup final: New Zealand 8-7 France". BBC Sport. Retrieved 4 December 2013.
                    363. ^ a b "New Zealand Cuisine". New Zealand Tourism Guide. January 2016. Retrieved 4 January 2016.
                    364. ^ Petrie, Hazel (November 2008). "Kai Pākehā – introduced foods". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 June 2017.
                    365. ^ Whaanga, Mere (June 2006). "Mātaitai – shellfish gathering". Te Ara: The Encyclopedia of New Zealand. Retrieved 27 June 2017.
                    366. ^ "Story: Shellfish". Te Ara: The Encyclopedia of New Zealand. Retrieved 29 August 2016.
                    367. ^ Burton, David (September 2013). "Cooking – Cooking methods". Te Ara: The Encyclopedia of New Zealand. Retrieved 11 December 2016.
                    368. ^ Royal, Charles; Kaka-Scott, Jenny (September 2013). "Māori foods – kai Māori". Te Ara: The Encyclopedia of New Zealand. Retrieved 1 September 2016.

                    References

                    • Alley, Roderic (2008). New Zealand in World Affairs IV 1990–2005. New Zealand: Victoria University Press. ISBN 978-0-864-73548-5.
                    • Bain, Carolyn (2006). New Zealand. Lonely Planet. ISBN 1-74104-535-5.
                    • Garden, Donald (2005). Stoll, Mark (ed.). Australia, New Zealand, and the Pacific: An Environmental History. Nature and Human Societies. ABC-CLIO/Greenwood. ISBN 978-1-57607-868-6.
                    • Hay, Jennifer; Maclagan, Margaret; Gordon, Elizabeth (2008). Dialects of English: New Zealand English. Edinburgh University Press. ISBN 978-0-7486-2529-1.
                    • Kennedy, Jeffrey (2007). "Leadership and Culture in New Zealand". In Chhokar, Jagdeep; Brodbeck, Felix; House, Robert (eds.). Culture and Leadership Across the World: The GLOBE Book of In-Depth Studies of 25 Societies. United States: Psychology Press. ISBN 978-0-8058-5997-3.
                    • King, Michael (2003). The Penguin History of New Zealand. New Zealand: Penguin Books. ISBN 978-0-14-301867-4.
                    • Mein Smith, Philippa (2005). A Concise History of New Zealand. Australia: Cambridge University Press. ISBN 0-521-54228-6.
                    • Smelt, Roselynn; Jui Lin, Yong (2009). New Zealand. Cultures of the World (2nd ed.). New York: Marshall Cavendish. ISBN 978-0-7614-3415-3.

                    Further reading

                    External links

                    Government
                    Travel
                    General Information
                    Retrieved from "https://en.wikipedia.org/w/index.php?title=New_Zealand&oldid=917976458"

                    Navigation menu

                    ================================================ FILE: test/test-pages/wikipedia-3/expected-metadata.json ================================================ { "title": "Hermitian matrix", "byline": "Contributors to Wikimedia projects", "dir": "ltr", "lang": "en", "excerpt": "In mathematics, a Hermitian matrix (or self-adjoint matrix) is a complex square matrix that is equal to its own conjugate transpose—that is, the element in the i-th row and j-th column is equal to the complex conjugate of the element in the j-th row and i-th column, for all indices i and j:", "siteName": "Wikimedia Foundation, Inc.", "publishedTime": "2003-02-28T21:51:08Z", "readerable": true } ================================================ FILE: test/test-pages/wikipedia-3/expected.html ================================================

                    In mathematics, a Hermitian matrix (or self-adjoint matrix) is a complex square matrix that is equal to its own conjugate transpose—that is, the element in the i-th row and j-th column is equal to the complex conjugate of the element in the j-th row and i-th column, for all indices i and j:

                    or in matrix form:

                    .

                    Hermitian matrices can be understood as the complex extension of real symmetric matrices.

                    If the conjugate transpose of a matrix is denoted by , then the Hermitian property can be written concisely as

                    Hermitian matrices are named after Charles Hermite, who demonstrated in 1855 that matrices of this form share a property with real symmetric matrices of always having real eigenvalues. Other, equivalent notations in common use are , although note that in quantum mechanics, typically means the complex conjugate only, and not the conjugate transpose.

                    Alternative characterizations[edit]

                    Hermitian matrices can be characterized in a number of equivalent ways, some of which are listed below:

                    Equality with the adjoint[edit]

                    A square matrix is Hermitian if and only if it is equal to its adjoint, that is, it satisfies

                    for any pair of vectors , where denotes the inner product operation.

                    This is also the way that the more general concept of self-adjoint operator is defined.

                    Reality of quadratic forms[edit]

                    A square matrix is Hermitian if and only if it is such that

                    Spectral properties[edit]

                    A square matrix is Hermitian if and only if it is unitarily diagonalizable with real eigenvalues.

                    Applications[edit]

                    Hermitian matrices are fundamental to the quantum theory of matrix mechanics created by Werner Heisenberg, Max Born, and Pascual Jordan in 1925.

                    Examples[edit]

                    In this section, the conjugate transpose of matrix is denoted as , the transpose of matrix is denoted as and conjugate of matrix is denoted as .

                    See the following example:

                    The diagonal elements must be real, as they must be their own complex conjugate.

                    Well-known families of Pauli matrices, Gell-Mann matrices and their generalizations are Hermitian. In theoretical physics such Hermitian matrices are often multiplied by imaginary coefficients,[1][2] which results in skew-Hermitian matrices (see below).

                    Here, we offer another useful Hermitian matrix using an abstract example. If a square matrix equals the multiplication of a matrix and its conjugate transpose, that is, , then is a Hermitian positive semi-definite matrix. Furthermore, if is row full-rank, then is positive definite.

                    Properties[edit]

                    [icon]

                    This section needs expansion with: Proof of the properties requested. You can help by adding to it. (February 2018)

                    • The entries on the main diagonal (top left to bottom right) of any Hermitian matrix are real.
                    Proof: By definition of the Hermitian matrix
                    so for i = j the above follows.
                    Only the main diagonal entries are necessarily real; Hermitian matrices can have arbitrary complex-valued entries in their off-diagonal elements, as long as diagonally-opposite entries are complex conjugates.
                    • A matrix that has only real entries is Hermitian if and only if it is symmetric. A real and symmetric matrix is simply a special case of a Hermitian matrix.
                    Proof: by definition. Thus Hij = Hji (matrix symmetry) if and only if (Hij is real).
                    • Every Hermitian matrix is a normal matrix. That is to say, AAH = AHA.
                    Proof: A = AH, so AAH = AA = AHA.
                    • The finite-dimensional spectral theorem says that any Hermitian matrix can be diagonalized by a unitary matrix, and that the resulting diagonal matrix has only real entries. This implies that all eigenvalues of a Hermitian matrix A with dimension n are real, and that A has n linearly independent eigenvectors. Moreover, a Hermitian matrix has orthogonal eigenvectors for distinct eigenvalues. Even if there are degenerate eigenvalues, it is always possible to find an orthogonal basis of ℂn consisting of n eigenvectors of A.
                    • The sum of any two Hermitian matrices is Hermitian.
                    Proof: as claimed.
                    • The inverse of an invertible Hermitian matrix is Hermitian as well.
                    Proof: If , then , so as claimed.
                    • The product of two Hermitian matrices A and B is Hermitian if and only if AB = BA.
                    Proof: Note that Thus if and only if .
                    Thus An is Hermitian if A is Hermitian and n is an integer.
                    • The Hermitian complex n-by-n matrices do not form a vector space over the complex numbers, ℂ, since the identity matrix In is Hermitian, but i In is not. However the complex Hermitian matrices do form a vector space over the real numbers ℝ. In the 2n2-dimensional vector space of complex n × n matrices over ℝ, the complex Hermitian matrices form a subspace of dimension n2. If Ejk denotes the n-by-n matrix with a 1 in the j,k position and zeros elsewhere, a basis (orthonormal w.r.t. the Frobenius inner product) can be described as follows:
                    together with the set of matrices of the form
                    and the matrices
                    where denotes the complex number , called the imaginary unit.
                    where are the eigenvalues on the diagonal of the diagonal matrix .
                    • The determinant of a Hermitian matrix is real:
                    Proof:
                    Therefore if .
                    (Alternatively, the determinant is the product of the matrix's eigenvalues, and as mentioned before, the eigenvalues of a Hermitian matrix are real.)

                    Decomposition into Hermitian and skew-Hermitian[edit]

                    Additional facts related to Hermitian matrices include:

                    • The sum of a square matrix and its conjugate transpose is Hermitian.
                    • The difference of a square matrix and its conjugate transpose is skew-Hermitian (also called antihermitian). This implies that the commutator of two Hermitian matrices is skew-Hermitian.
                    • An arbitrary square matrix C can be written as the sum of a Hermitian matrix A and a skew-Hermitian matrix B. This is known as the Toeplitz decomposition of C.[3]:p. 7

                    Rayleigh quotient[edit]

                    In mathematics, for a given complex Hermitian matrix M and nonzero vector x, the Rayleigh quotient[4] , is defined as:[3]:p. 234[5]

                    .

                    For real matrices and vectors, the condition of being Hermitian reduces to that of being symmetric, and the conjugate transpose to the usual transpose . Note that for any non-zero real scalar . Also, recall that a Hermitian (or real symmetric) matrix has real eigenvalues.

                    It can be shown[citation needed] that, for a given matrix, the Rayleigh quotient reaches its minimum value (the smallest eigenvalue of M) when is (the corresponding eigenvector). Similarly, and .

                    The Rayleigh quotient is used in the min-max theorem to get exact values of all eigenvalues. It is also used in eigenvalue algorithms to obtain an eigenvalue approximation from an eigenvector approximation. Specifically, this is the basis for Rayleigh quotient iteration.

                    The range of the Rayleigh quotient (for matrix that is not necessarily Hermitian) is called a numerical range (or spectrum in functional analysis). When the matrix is Hermitian, the numerical range is equal to the spectral norm. Still in functional analysis, is known as the spectral radius. In the context of C*-algebras or algebraic quantum mechanics, the function that to M associates the Rayleigh quotient R(M, x) for a fixed x and M varying through the algebra would be referred to as "vector state" of the algebra.

                    See also[edit]

                    References[edit]

                    1. ^ Frankel, Theodore (2004). The Geometry of Physics: an introduction. Cambridge University Press. p. 652. ISBN 0-521-53927-7.
                    2. ^ Physics 125 Course Notes at California Institute of Technology
                    3. ^ a b Horn, Roger A.; Johnson, Charles R. (2013). Matrix Analysis, second edition. Cambridge University Press. ISBN 9780521839402.
                    4. ^ Also known as the Rayleigh–Ritz ratio; named after Walther Ritz and Lord Rayleigh.
                    5. ^ Parlet B. N. The symmetric eigenvalue problem, SIAM, Classics in Applied Mathematics,1998

                    External links[edit]

                    ================================================ FILE: test/test-pages/wikipedia-3/source.html ================================================ Hermitian matrix - Wikipedia

                    Hermitian matrix

                    From Wikipedia, the free encyclopedia
                    Jump to navigation Jump to search
                    For matrices with symmetry over the real number field, see symmetric matrix.

                    In mathematics, a Hermitian matrix (or self-adjoint matrix) is a complex square matrix that is equal to its own conjugate transpose—that is, the element in the i-th row and j-th column is equal to the complex conjugate of the element in the j-th row and i-th column, for all indices i and j:

                    or in matrix form:

                    .

                    Hermitian matrices can be understood as the complex extension of real symmetric matrices.

                    If the conjugate transpose of a matrix is denoted by , then the Hermitian property can be written concisely as

                    Hermitian matrices are named after Charles Hermite, who demonstrated in 1855 that matrices of this form share a property with real symmetric matrices of always having real eigenvalues. Other, equivalent notations in common use are , although note that in quantum mechanics, typically means the complex conjugate only, and not the conjugate transpose.

                    Alternative characterizations[edit]

                    Hermitian matrices can be characterized in a number of equivalent ways, some of which are listed below:

                    Equality with the adjoint[edit]

                    A square matrix is Hermitian if and only if it is equal to its adjoint, that is, it satisfies

                    for any pair of vectors , where denotes the inner product operation.

                    This is also the way that the more general concept of self-adjoint operator is defined.

                    Reality of quadratic forms[edit]

                    A square matrix is Hermitian if and only if it is such that

                    Spectral properties[edit]

                    A square matrix is Hermitian if and only if it is unitarily diagonalizable with real eigenvalues.

                    Applications[edit]

                    Hermitian matrices are fundamental to the quantum theory of matrix mechanics created by Werner Heisenberg, Max Born, and Pascual Jordan in 1925.

                    Examples[edit]

                    In this section, the conjugate transpose of matrix is denoted as , the transpose of matrix is denoted as and conjugate of matrix is denoted as .

                    See the following example:

                    The diagonal elements must be real, as they must be their own complex conjugate.

                    Well-known families of Pauli matrices, Gell-Mann matrices and their generalizations are Hermitian. In theoretical physics such Hermitian matrices are often multiplied by imaginary coefficients,[1][2] which results in skew-Hermitian matrices (see below).

                    Here, we offer another useful Hermitian matrix using an abstract example. If a square matrix equals the multiplication of a matrix and its conjugate transpose, that is, , then is a Hermitian positive semi-definite matrix. Furthermore, if is row full-rank, then is positive definite.

                    Properties[edit]

                    • The entries on the main diagonal (top left to bottom right) of any Hermitian matrix are real.
                    Proof: By definition of the Hermitian matrix
                    so for i = j the above follows.
                    Only the main diagonal entries are necessarily real; Hermitian matrices can have arbitrary complex-valued entries in their off-diagonal elements, as long as diagonally-opposite entries are complex conjugates.
                    • A matrix that has only real entries is Hermitian if and only if it is symmetric. A real and symmetric matrix is simply a special case of a Hermitian matrix.
                    Proof: by definition. Thus Hij = Hji (matrix symmetry) if and only if (Hij is real).
                    • Every Hermitian matrix is a normal matrix. That is to say, AAH = AHA.
                    Proof: A = AH, so AAH = AA = AHA.
                    • The finite-dimensional spectral theorem says that any Hermitian matrix can be diagonalized by a unitary matrix, and that the resulting diagonal matrix has only real entries. This implies that all eigenvalues of a Hermitian matrix A with dimension n are real, and that A has n linearly independent eigenvectors. Moreover, a Hermitian matrix has orthogonal eigenvectors for distinct eigenvalues. Even if there are degenerate eigenvalues, it is always possible to find an orthogonal basis of ℂn consisting of n eigenvectors of A.
                    • The sum of any two Hermitian matrices is Hermitian.
                    Proof: as claimed.
                    • The inverse of an invertible Hermitian matrix is Hermitian as well.
                    Proof: If , then , so as claimed.
                    • The product of two Hermitian matrices A and B is Hermitian if and only if AB = BA.
                    Proof: Note that Thus if and only if .
                    Thus An is Hermitian if A is Hermitian and n is an integer.
                    • For an arbitrary complex valued vector v the product is real because of . This is especially important in quantum physics where Hermitian matrices are operators that measure properties of a system e.g. total spin which have to be real.
                    • The Hermitian complex n-by-n matrices do not form a vector space over the complex numbers, ℂ, since the identity matrix In is Hermitian, but i In is not. However the complex Hermitian matrices do form a vector space over the real numbers ℝ. In the 2n2-dimensional vector space of complex n × n matrices over ℝ, the complex Hermitian matrices form a subspace of dimension n2. If Ejk denotes the n-by-n matrix with a 1 in the j,k position and zeros elsewhere, a basis (orthonormal w.r.t. the Frobenius inner product) can be described as follows:
                    together with the set of matrices of the form
                    and the matrices
                    where denotes the complex number , called the imaginary unit.
                    • If n orthonormal eigenvectors of a Hermitian matrix are chosen and written as the columns of the matrix U, then one eigendecomposition of A is where and therefore
                    where are the eigenvalues on the diagonal of the diagonal matrix .
                    • The determinant of a Hermitian matrix is real:
                    Proof:
                    Therefore if .
                    (Alternatively, the determinant is the product of the matrix's eigenvalues, and as mentioned before, the eigenvalues of a Hermitian matrix are real.)

                    Decomposition into Hermitian and skew-Hermitian[edit]

                    Additional facts related to Hermitian matrices include:

                    • The sum of a square matrix and its conjugate transpose is Hermitian.
                    • The difference of a square matrix and its conjugate transpose is skew-Hermitian (also called antihermitian). This implies that the commutator of two Hermitian matrices is skew-Hermitian.
                    • An arbitrary square matrix C can be written as the sum of a Hermitian matrix A and a skew-Hermitian matrix B. This is known as the Toeplitz decomposition of C.[3]:p. 7

                    Rayleigh quotient[edit]

                    Main article: Rayleigh quotient

                    In mathematics, for a given complex Hermitian matrix M and nonzero vector x, the Rayleigh quotient[4] , is defined as:[3]:p. 234[5]

                    .

                    For real matrices and vectors, the condition of being Hermitian reduces to that of being symmetric, and the conjugate transpose to the usual transpose . Note that for any non-zero real scalar . Also, recall that a Hermitian (or real symmetric) matrix has real eigenvalues.

                    It can be shown[citation needed] that, for a given matrix, the Rayleigh quotient reaches its minimum value (the smallest eigenvalue of M) when is (the corresponding eigenvector). Similarly, and .

                    The Rayleigh quotient is used in the min-max theorem to get exact values of all eigenvalues. It is also used in eigenvalue algorithms to obtain an eigenvalue approximation from an eigenvector approximation. Specifically, this is the basis for Rayleigh quotient iteration.

                    The range of the Rayleigh quotient (for matrix that is not necessarily Hermitian) is called a numerical range (or spectrum in functional analysis). When the matrix is Hermitian, the numerical range is equal to the spectral norm. Still in functional analysis, is known as the spectral radius. In the context of C*-algebras or algebraic quantum mechanics, the function that to M associates the Rayleigh quotient R(M, x) for a fixed x and M varying through the algebra would be referred to as "vector state" of the algebra.

                    See also[edit]

                    References[edit]

                    1. ^ Frankel, Theodore (2004). The Geometry of Physics: an introduction. Cambridge University Press. p. 652. ISBN 0-521-53927-7.
                    2. ^ Physics 125 Course Notes at California Institute of Technology
                    3. ^ a b Horn, Roger A.; Johnson, Charles R. (2013). Matrix Analysis, second edition. Cambridge University Press. ISBN 9780521839402.
                    4. ^ Also known as the Rayleigh–Ritz ratio; named after Walther Ritz and Lord Rayleigh.
                    5. ^ Parlet B. N. The symmetric eigenvalue problem, SIAM, Classics in Applied Mathematics,1998

                    External links[edit]

                    Retrieved from "https://en.wikipedia.org/w/index.php?title=Hermitian_matrix&oldid=942460710"

                    Navigation menu

                    ================================================ FILE: test/test-pages/wikipedia-4/expected-metadata.json ================================================ { "title": "List of films featuring time loops", "byline": "Contributors to Wikimedia projects", "dir": "ltr", "lang": "en", "excerpt": "From Wikipedia, the free encyclopedia", "siteName": "Wikimedia Foundation, Inc.", "publishedTime": "2014-03-27T19:11:24Z", "readerable": true } ================================================ FILE: test/test-pages/wikipedia-4/expected.html ================================================

                    From Wikipedia, the free encyclopedia

                    This list of films featuring time loops in which characters experience the same period of time which is repeatedly resetting: when a certain condition is met, such as a death of a character or a clock reaches a certain time, the loop starts again, with one or more characters retaining the memories from the previous loop.[1]: 207  The list provides the names and brief synopses of films in which time loops are a prominent plot device.

                    For a list of films that include any kind of time travel (including time loops) see time travel in films.

                    Films with time loops
                    Film Year Description
                    Repeat Performance 1947 A woman who shot her husband on New Year's Eve in 1946 wishes she could live the year all over again and somehow gets her chance, which leads her to try and stop the events of the past, to futile results.[2]
                    The Time Travelers 1964 Scientists test a time-viewing device but the screen becomes a portal which they enter; then it disappears, stranding them in a future devastated by nuclear war. [3]
                    Journey to the Center of Time 1967 Remake of The Time Travelers[4]
                    Le 15 Mai 1969 A couple wakes up one sunny morning on the 15th of May. Apparently, they had the same nightmare and the day unfolds strangely. The next day, we are still on the 15th of May. Claire Denis's graduation work from La Fémis.[5]
                    The Girl Who Leapt Through Time 1983 Based on Yasutaka Tsutsui's 1965 novel of the same name. The story is about a Japanese schoolgirl who accidentally gains the ability to time travel, which she experiments with and attempts to alter past events, leading her on a journey through multiple time loops.[6][7]
                    Urusei Yatsura 2: Beautiful Dreamer 1984 The students of Tomobiki high school relive the day before the school festival over and over.[8][9][10]
                    Mirror for a Hero 1987 Zerkalo dlya geroya, a Soviet Union film by Vladimir Khotinenko. Two heroes circle multiple times in 1949, adapting to the harsh post-war life, meeting the parents of one of them, finally returning to the present time, having reassessed their attitude towards elders.[11][12]
                    12:01 PM 1990 This short film is the first film adaptation of the short story "12:01 PM" by Richard A. Lupoff, which was published in 1973 in The Magazine of Fantasy and Science Fiction. It originally aired on cable television as part of the Showtime 30-Minute Movie anthology series. It was nominated for an Academy Award.[13]
                    12:01 1993 The second film adaptation of the short story "12:01 PM" by Richard A. Lupoff, which was published in 1973 in The Magazine of Fantasy and Science Fiction. Office worker Barry Thomas is forced to relive the worst day of his life.[14]
                    Groundhog Day 1993 Self-centered television weatherman Phil Connors (Bill Murray) is forced to relive the same day over and over.[15] This movie is generally seen as the quintessential time-loop movie by many with its name being synonymous with the genre as a whole.[16][17][18]
                    Christmas Every Day 1996 An American television movie based on William Dean Howells' 1892 short story "Christmas Every Day". A selfish teenager is forced to relive the same Christmas every day.[19]
                    Retroactive 1997 A psychiatrist makes multiple trips through time to save a woman who was murdered by her husband.[20]
                    Run Lola Run 1998 The story follows a woman named Lola (Franka Potente) who needs to obtain 100,000 Deutschmarks in twenty minutes to save the life of her boyfriend by resetting time multiple times by 20 minutes.[21]
                    Mickey's Once Upon a Christmas 1999 Huey, Dewey, and Louie must repeat Christmas Day until they come to learn the true meaning of Christmas.[22]
                    Naken 2000 On the day of his wedding, Anders wakes up naked in an elevator. As he attempts to piece together the reason for his predicament, the day repeats itself, and he becomes stuck in a time loop.[23]
                    Camp Slaughter 2005 A group of present-day teenagers are sent back to 1981, and discover a summer camp which is stuck reliving the day a demented killer went on a rampage.[24]
                    Christmas Do-Over 2006 A father is stuck in a time loop when his son wishes that every day was Christmas day. A remake of Christmas Every Day from an adult perspective.
                    The Girl Who Leapt Through Time 2006 Anime based on Yasutaka Tsutsui's 1965 novel of the same name.
                    Salvage 2006 Claire encounters horror and murder after finishing a day working at a convenience store, not once but many times. An official selection of the 2006 Sundance Festival.[25]
                    The Last Day of Summer 2007 11-year-old Luke gets his wish that every day could be the last day of summer.[26]
                    Frequently Asked Questions About Time Travel 2009 Three social misfits navigate a time-travel conundrum originating from their local pub.[27]
                    Triangle 2009 A group of friends go on a boat trip and become stranded due to a storm. They discover a cruise ship which causes them to experience a series of repeating events.[28]
                    The Girl Who Leapt Through Time 2010 Third film based on Yasutaka Tsutsui's 1965 novel of the same name.
                    Repeaters 2010 A group of inmates at a rehabilitation facility are forced to repeat the same day over and over.[29]
                    Source Code 2011 U.S. Army Aviation pilot Captain Colter Stevens repeatedly experiences the last eight minutes of another person's life to identify the bomber in a terrorist attack in order to prevent a second, nuclear attack on Chicago.[30]
                    The Road 2011 A group of teens finally work up the fortitude to explore a rural road, that has all the warning signs of being a bad idea.[31]
                    12 Dates of Christmas 2011 Kate finds herself reliving Christmas Eve (including a blind date with a man named Miles) over and over. She must discover how to break the cycle – should she attempt to win back her ex-boyfriend Jack, should she pursue Miles, or something else?[32]
                    Puella Magi Madoka Magica: The Movie 2012 Homura Akemi has the ability to travel 30 days back in time, and has done so countless times. Each month, Homura fails in her attempts to save Madoka Kaname from destruction, and is forced to relive the tragedy of losing Madoka.[33]
                    Mine Games 2012 A group of friends vacationing in the woods discover their own corpses and realize they are trapped in a time loop that they attempt to break.[34]
                    Blood Punch 2013 A trio visits a remote cabin for a one-time criminal endeavor and finds each member independently reliving the same day.[35]
                    Haunter 2013 The ghost of a teenager, Lisa Johnson (Abigail Breslin), who, along with her family, keeps reliving the same day (although she is the only one aware of it) tries to protect a young girl, Olivia (Eleanor Zichy), and her family from a dead serial killer, the Pale Man (Stephen McHattie), who can possess the living.[36]
                    El Ascensor 2014 On their fifth wedding anniversary, Sitio and Ana start arguing inside an elevator that keeps opening in their same floor. Emotionally and physically trapped, the two will have to work together to find a way out.[37]
                    One-Minute Time Machine 2014 This 6-minute comedic short film directed by Devon Avery and featuring Brian Dietzen and Erinn Hayes, parodies the time loop genre (and itself) when repeated applications of the device are used to redo a botched romantic approach.[38]
                    Oru Kanniyum Moonu Kalavaanikalum 2014 A group of friends find themselves in a time loop with 3 different time slots for the same incident and the result will vary due to the time and the result of people mood.[39]
                    Premature 2014 Rob, a high school senior, has to relive losing his virginity over and over until he gets it right, with the right girl.[40]
                    The Incident 2014 Two stories of groups trapped in infinite time and space loops. One, an endless staircase, the other, an endless road, both looping back on themselves. While time resets in their surroundings each day, the trapped characters continue to age within the loops.[41]
                    The Infinite Man 2014 After a romantic weekend break with his girlfriend goes awry, a scientist invents time travel in order to create for her the perfect getaway, but inadvertently traps her in a recurring temporal loop.[42]
                    Edge of Tomorrow 2014 Maj. William "Bill" Cage (Tom Cruise) and Special Forces soldier Rita Vrataski (Emily Blunt) team up to fight a hostile alien race known as Mimics, with Cage continually returning to a repeating battle through a time loop.[43]
                    Magnetic 2015 Alice is stuck in a time loop, repeating each week, trying to find a way to prevent the Earth from being destroyed by a massive solar flare.[44]
                    The Final Girls 2015 A group of friends find themselves trapped in an '80s horror movie that restarts every 92 minutes.[45]
                    ARQ 2016 An engineer, whose invention causes time to loop during a home invasion, attempts to save his former lover while learning who has targeted him and why.[46]
                    Doctor Strange 2016 Aided by the powers of The Eye of Agamotto, Stephen Strange traps himself and Dormammu in a time loop in order to bargain against him consuming the Earth.[47]
                    Erased 2016 A Japanese young man travels back in time to prevent the murder of his mother and others, looping through different versions of reality as his actions change the past.[48][49] (The same basic story exists in manga, anime, live action movie, and live action TV series versions.)
                    Groundhog Day for a Black Man 2016 Short film about a black man who keeps reliving the same day in which he ends up being shot by a police officer.[50] The director of this film accused Netflix's Two Distant Strangers of plagiarism.[51]
                    Miss Peregrine's Home for Peculiar Children 2016 Teenager Jake visits the ruins of an orphanage that was destroyed by a bombing raid during World War II and discovers that the original inhabitants survived by creating a time loop set to repeat the day of September 3, 1943.[52]
                    Naked 2017 Rob is caught in a time loop as he keeps waking up naked in a hotel elevator on the day of his wedding.[23] This is a remake of the 2000 Swedish film Naken.
                    Happy Death Day 2017 A college student is murdered on her birthday, Monday the 18th. She wakes up the morning of the 18th, alive, facing the same day over and over, while also getting weaker. She must figure out who her killer is in order to escape the loop.[53]
                    Before I Fall 2017 A popular high school senior finds herself reliving the same day over and over.[54]
                    The Endless 2017 Two brothers return to a cult that is stuck in a time loop.[55]
                    A Day 2017 A father attempts to save his daughter whilst stuck in an endless time loop.[56]
                    Reset 2017 A physicist uses an experimental wormhole machine to travel to parallel universes and back in time by an hour, trying to save her kidnapped son.[57]
                    The Fare 2018 A taxi driver and his passenger find themselves locked in an endless time loop as they have to repeat their journey over and over.[58]
                    When We First Met 2018 A guy’s grief over losing Ms. Right, grief he recovers from by time traveling to earn that magic “reset.” [59]
                    Koko-di Koko-da 2019 A struggling couple on a camping trip is caught in a time loop where they are tormented by a group of fairy-tale characters.[60]
                    Game Over 2019 Swapna, a wheelchair user, is defending her home from a mysterious intruder. Every time she is killed by the intruder, she awakens to the start of the day. However, the concept is more along the lines of how a video game works.[61]
                    The Obituary of Tunde Johnson 2019 A gay African-American teenager repeatedly relives the day of his fatal shooting in an unprovoked police brutality incident.[62]
                    Happy Death Day 2U 2019 Immediately after the events of the first film (Happy Death Day), Tree Gelbman (Jessica Rothe) unexpectedly re-enters the time loop except this time in an alternate reality.[63][64]
                    See You Yesterday 2019 An ambitious science prodigy uses her prowess and capabilities to create time machines, in order to save her brother who has been killed by a police officer. As she tries to alter the events of the past, she will eventually face the perilous consequences of time travel.[65]
                    Volition 2019 A man afflicted with clairvoyance tries to change his fate when a series of events leads to a vision of his own imminent murder.[66]
                    Two Distant Strangers 2020 An American short science-fiction film that examines the deaths of black Americans during encounters with police through the eyes of a character trapped in a time loop that keeps ending in his death.[67]
                    Palm Springs 2020 When carefree Nyles and reluctant maid of honor Sarah have a chance encounter at a Palm Springs wedding, things get complicated the next morning when they find themselves unable to escape the venue, themselves, or each other.[68]
                    Boss Level 2021 A retired special forces officer is trapped in a time loop on the day of his death.[69]
                    Christmas...Again?! 2021 A holiday Disney Channel Original Movie starring Scarlett Estevez as Ro, whose Christmas wish goes awry as she finds herself living christmas day over and over again.[70]
                    Churuli 2021 The film, directed by Lijo Jose Pellissery, tells the story of two undercover police officers who go in search of a wanted criminal named Mayiladumparambil Joy. However, they get stuck in the village of Churuli as eerie mysteries unfold.

                    The original story and characters are based on a short story from the book Kaligaminarile Kuttavalikal written by Vinoy Thomas.[71][72]

                    Jango 2021 A neurosurgeon who is stuck in a time loop tries to save his estranged wife from being killed by a mystery man.[73]
                    Maanaadu 2021 A man who is stuck in a time loop tries to stop a political rally from taking place in order to save the state's chief minister and prevent religious violence.[74]
                    The Map of Tiny Perfect Things 2021 Two teens live the same day repeatedly, enabling them to create the titular map.[75]
                    Meet Cute 2022 A Manhattan-native, Sheila, discovers a time machine in a nail salon and uses it to continually fix elements of a date she had the previous night.[76]
                    Mondays: see you "this" week 2022 The staff of an advertising agency office are stuck in a time loop except their boss.[77]
                    Vandits 2022 A group of incompetent criminals tries to rob a bingo hall, only to get trapped in a time loop in which they repeatedly stage the heist but fail in a different way each time.[78]
                    Zanox 2022 Misi, a high school student, relives his graduation day after he discovers his experimental medication has a weird side effect.[79]
                    Dreadful Chapters 2023 Indian Malayalam-language horror film directed by Nirmal Baby Varghese.[80]

                    See also[edit]

                    References[edit]

                    1. ^ Matthew Jones; Joan Ormrod (2015), Time Travel in Popular Media, McFarland & Company, ISBN 9780786478071
                    2. ^ "In the Loop: Repeat Performance" . 24 August 2021.
                    3. ^ Bogue, Mike (September 26, 2017). Apocalypse Then: American and Japanese Atomic Cinema, 1951-1967. McFarland & Company. p. 80. ISBN 978-1476668413.
                    4. ^ Bogue, Mike (September 26, 2017). Apocalypse Then: American and Japanese Atomic Cinema, 1951-1967. McFarland & Company. p. 80. ISBN 978-1476668413.
                    5. ^ "Le 15 Mai" .
                    6. ^ "THE GIRL WHO LEAPT THROUGH TIME (2006)" . Deptford Cinema. August 9, 2017. Retrieved 27 January 2020.
                    7. ^ "THE GIRL WHO LEAPT THROUGH TIME (2006) at Deptford Cinema" . TicketSource. 9 August 2017. Retrieved 27 January 2020.
                    8. ^ Hartl, John (September 3, 1998). "Disney Will Distribute Animated Japanese Movie" . The Seattle Times. Retrieved May 15, 2017.
                    9. ^ French, Todd (July 1996). "Reviews" . Cinefantastique. 27 (11/12): 123. Retrieved April 28, 2017. The student body of Tomobiki High experiences a major cause of deja vu when they find themselves endlessly repeating "the day before the School Festival."
                    10. ^ "Oshii says dissatisfaction behind 'eternal youth'". Yomiuri Shimbun. August 1, 2008. p. 18. The sequel, 1984's Urusei Yatsura 2: Beautiful Dreamer, meanwhile, included a gut-wrenching philosophical aspect in the storyline. In the movie, the characters relive the night before a school festival over and over, eventually blurring the line between reality and fantasy. The concept and execution took that film beyond mere anime geared toward kids.
                    11. ^ Bratersky, Alexander (29 July 2013). "'Alterum' Dance Troupe to Conquer Middle East" . The Moscow Times. Retrieved 1 May 2015.
                    12. ^ Keller, Bill; Times, Special to the New York (23 April 1988). "A Movie Tribute for Stalin Generation" . The New York Times. Retrieved 1 May 2015.
                    13. ^ "1990 (63rd): Short Film (Live Action)" . Academy Awards Database. Academy of Motion Picture Arts and Sciences. Archived from the original on 2015-04-02.
                    14. ^ "It's like Groundhog Day, but different. What is it?" . TV Guide. Archived from the original on 2014-03-28. Retrieved 2014-03-27.
                    15. ^ Fischer, Russ (2011-02-02). "'Groundhog Day:' Just How Long Was Phil Connors Stuck in a Time Loop?" . Slashfilm. Retrieved 2014-03-27.
                    16. ^ "Over and over: why is Hollywood still obsessed with Groundhog Day?" . the Guardian. 2019-02-13. Retrieved 2022-03-23.
                    17. ^ "Groundhog Day: the perfect comedy, for ever" . the Guardian. 2013-02-07. Retrieved 2022-03-23.
                    18. ^ "The Pop Culture Legacy of 'Groundhog Day'" . www.vice.com. 2 February 2017. Retrieved 2022-03-23.
                    19. ^ Caro, Jason. "Christmas Every Day" . Radio Times. Retrieved 6 May 2014.
                    20. ^ Bailey, Keith. "Retroactive" . Radio Times. Retrieved 2015-11-23.
                    21. ^ Cosmicneed, Eden (22 June 2021). "In the Loop: Run Lola Run" . Cosmic Need. Retrieved 19 April 2023.
                    22. ^ Rain Fordem, Kajsa (24 December 2015). "Rotoscopers' 12 Days of Christmas: 'The Polar Express'" . Rotoscopers. Retrieved 12 September 2017.
                    23. ^ Jump up to: a b Fleming, Mike Jr. (21 September 2016). "Netflix Gets 'Naked' With Marlon Wayans & Regina Hall" . Deadline Hollywood. Retrieved 18 May 2017.
                    24. ^ Wheeler, Jeremy (2014). "Camp Slaughter (2005)" . Movies & TV Dept. The New York Times. Archived from the original on 12 May 2014. Retrieved 12 May 2014.
                    25. ^ Chang, Justin (31 January 2006). "Review: 'Salvage'" . Variety. Retrieved 6 May 2014.
                    26. ^ Gutierrez, Reuben (2007). "The Last Day of Summer DVD Review" . DVDizzy.com. Retrieved 6 May 2014.
                    27. ^ "Frequently Asked Questions About Time Travel (2009)" . Rotten Tomatoes. Retrieved 2024-02-02.
                    28. ^ Li, Shirley (2009-10-01). "Triangle Review" . Empire. Retrieved 2018-07-09.
                    29. ^ Alex Billington (2010-08-31). "Watch: Carl Bessai's Time Loop Sci-Fi Film 'Repeaters' Trailer" . First Showing. Retrieved 2014-03-28.
                    30. ^ Lawler, Kelley (6 June 2014). "It's still 'Groundhog Day': The 4 best time-loop movies" . USA Today. Retrieved 26 September 2017.
                    31. ^ Catsoulis, Jeannette. "Crawling Backwards In Time Into Haunted Real Estate" . New York Times. Retrieved 2015-11-23.
                    32. ^ Griffiths, John (6 December 2011). "12 Dates of Christmas" . US Weekly. Retrieved 7 May 2014.
                    33. ^ Loveridge, Lynzee (2013-08-10). "8 Essential Time-Travelling Heroines" . Anime News Network.
                    34. ^ Bellete, Kwenton (17 August 2012). "MIFF 2012 Review: MINE GAMES world premiere" . Screen Anarchy. Retrieved 26 September 2017.
                    35. ^ "Blood Punch - Fruity Looper" . 9 March 2017. Retrieved 22 November 2018.
                    36. ^ Rotten, Doc. "Film Review: Haunter (2013)" . HorrorNews. Retrieved 23 May 2014.
                    37. ^ Time Loop Movies – A Definitive 2023 List (Live The Same Day Over)
                    38. ^ Morgan, Nigel (May 12, 2020). "Review: One-Minute Time Machine" . Retrieved August 17, 2023.
                    39. ^ "Movie Review : Oru Kanniyum Moonu Kalavaanikalum" . Sify. Archived from the original on 2014-04-06. Retrieved 2014-04-15.
                    40. ^ Leydon, Joe (2 July 2014). "Film Review: 'Premature'" . Variety. Retrieved 31 August 2014.
                    41. ^ Film Review: ‘The Incident’
                    42. ^ Hedone. "Hedone / The Infinite Man" . Retrieved 24 Feb 2018.
                    43. ^ Gettell, Oliver (2014-03-25). "'Edge of Tomorrow' trailer: Tom Cruise fights an eternal battle" . Los Angeles Times. Retrieved 2014-03-28.
                    44. ^ "Another Hole In The Head Film Festival Reviews - Beyond Chron" . beyondchron.org. 5 November 2015. Retrieved May 24, 2018.
                    45. ^ Harvey, Dennis (March 15, 2015). "SXSW Film Review: 'The Final Girls'" . Variety.
                    46. ^ Tsoumbakos, Rachel (2016-09-01). "The Latest Netflix Original Movie 'ARQ' Is A Dystopian Time Loop Adventure" . The Inquisitr. Retrieved 2016-10-26.
                    47. ^ Couch, Aaron (26 November 2016). "'Doctor Strange' Writer on Time-Bending Twist and What It Means for Marvel's Future" . The Hollywood Reporter. Retrieved 21 April 2018.
                    48. ^ "ERASED" . Anime News Network. January 8, 2016. Retrieved February 14, 2016.
                    49. ^ "僕だけがいない街" . eiga.com. Retrieved 5 May 2022.
                    50. ^ Costello, Colin (2017-10-20). "'Groundhog Day for a Black Man' wows fest crowd" . Reel Chicago. Retrieved 2021-06-16.
                    51. ^ Frias, Lauren (2021-05-06). "An Oscar-winning filmmaker responded to accusations that his short copied a 2016 film about police brutality: 'The concept of the time loop film device is not new'" . Insider. Retrieved 2021-06-16.
                    52. ^ Hoffman, Jordan (September 25, 2016). "Miss Peregrine's Home For Peculiar Children review: mordant British YA X-Men is Tim Burton's best in 20 years" . The Guardian.
                    53. ^ Hall, Jacob (15 June 2017). "'Happy Death Day' Trailer: 'Groundhog Day' With a Lot More Murder" . /Film. Retrieved 16 June 2017.
                    54. ^ Felperin, Leslie (23 January 2017). "'Before I Fall': Film Review" . The Hollywood Reporter. Retrieved 15 February 2017.
                    55. ^ Crucchiola, Jordan (April 6, 2018). "Unpacking That Big Twist in The Endless" . Vulture.com. Retrieved February 19, 2020.
                    56. ^ Lee, Hyo-won (7 July 2017). "Cannes: South Korean Time Travel Film 'A Day' Sells to Multiple Territories" . The Hollywood Reporter. Retrieved 25 September 2017.
                    57. ^ "'Reset' ('Ni Shi Ying Jiu'): Film Review" . The Hollywood Reporter. 15 June 2017. Retrieved 3 March 2019.
                    58. ^ Harvey, Rupert (22 November 2019). "Movie Review: The Fare (2019)" . The Critical Movie Critics. Retrieved 22 December 2019.
                    59. ^ Moore, Roger. "Netflixable? Can this couple recapture the magic of "When We First Met?"" . Movie Nation. Retrieved 19 June 2018.
                    60. ^ Aguilar, Carlos (February 4, 2019). "Time Loop Nightmare 'Koko-di Koko-da' Haunts A Couple In Crisis [Sundance Review]" . The Playlist. Retrieved November 8, 2020.
                    61. ^ Lakshmi, Venkateswaran (June 12, 2019). "Game Over Movie Review" . The Times of India. Retrieved October 19, 2020.
                    62. ^ Beandrea July, "'The Obituary of Tunde Johnson': Film Review | TIFF 2019" . The Hollywood Reporter, September 8, 2019.
                    63. ^ Ouelette, Jennifer (1 December 2018). "A Pesky Time Loop Meets the Multiverse in Happy Death Day 2U First Trailer" . /Film. Retrieved 1 December 2018.
                    64. ^ "Happy Death Day 2U Review" . ScreenRant. 2019-02-13. Retrieved 2020-10-20.
                    65. ^ "'See You Yesterday' Review: An All-Too-Real Time-Travel Fantasy" . The New York Times. 2019-05-17. Retrieved 2020-11-19.
                    66. ^ Debruge, Peter (July 8, 2020). "'Palm Springs': Film Review" . Hollywood Reporter. Retrieved Jul 8, 2020.
                    67. ^ "Two Distant Strangers' Makes Sure Oscar Voters Don't Forget George Floyd" . 2021-03-08. Retrieved 2021-04-15.
                    68. ^ Debruge, Peter (January 28, 2020). "'Palm Springs': Film Review" . Variety. Retrieved Jul 3, 2020.
                    69. ^ Wiseman, Andreas (November 23, 2020). "Hulu Strikes U.S. Deal For Frank Grillo Action Sci-Fi Movie 'Boss Level'" . Deadline. Retrieved November 26, 2020.
                    70. ^ "Disney+ Release Date Revealed For Disney Channel Original Movie "Christmas Again"" . Disney Plus Informer. 28 October 2021. Retrieved 4 January 2022.
                    71. ^ "CHURULI aka LABYRINTH Explanation" . 20 November 2021.
                    72. ^ "Churuli and other Films based on Time Loops (Malayalam/Mathrubhumi Report)" .
                    73. ^ "Jango review: Watch it for the time loop angle" . Sify. Archived from the original on November 19, 2021. Retrieved 19 November 2021.
                    74. ^ "Maanaadu Movie Review: A riveting take on time loop underlined by clever writing" . Cinema Express. Retrieved 2021-11-25.
                    75. ^ Kenigsberg, Ben (2021-02-11). "'The Map of Tiny Perfect Things' Review: Another Do-Over" . The New York Times. Retrieved 2021-02-12.
                    76. ^ D'Alessandro, Anthony (16 August 2022). "Kaley Cuoco & Pete Davidson Romantic Comedy 'Meet Cute' Sets Fall Premiere Date On Peacock" . Deadline. Retrieved 30 August 2022.
                    77. ^ Hadfield, James (October 6, 2022). "'Mondays: See You "This" Week!': Zany office comedy gets stuck on repeat" . The Japan Times. Retrieved October 30, 2022.
                    78. ^ John Kirk, "Vandits: Wacky Canuck Heist Film Vulgar, Lowbrow… and Awesome" . Original Cin, December 14, 2022.
                    79. ^ "Criticism of Zanox - Risks and Side Effects - Fresno 24 English" . fresno24.com. 2022-05-22. Retrieved 2022-06-20.
                    80. ^ "Dreadful Chapters: Nirmal Varghese unveils first-look poster of his horror film" . Madhyamam. 19 July 2023. Retrieved 20 July 2022.
                    ================================================ FILE: test/test-pages/wikipedia-4/source.html ================================================ List of films featuring time loops - Wikipedia
                    Jump to content

                    List of films featuring time loops

                    From Wikipedia, the free encyclopedia
                    This is a dynamic list and may never be able to satisfy particular standards for completeness. You can help by adding missing items with reliable sources.

                    This list of films featuring time loops in which characters experience the same period of time which is repeatedly resetting: when a certain condition is met, such as a death of a character or a clock reaches a certain time, the loop starts again, with one or more characters retaining the memories from the previous loop.[1]: 207  The list provides the names and brief synopses of films in which time loops are a prominent plot device.

                    For a list of films that include any kind of time travel (including time loops) see time travel in films.

                    Films with time loops
                    Film Year Description
                    Repeat Performance 1947 A woman who shot her husband on New Year's Eve in 1946 wishes she could live the year all over again and somehow gets her chance, which leads her to try and stop the events of the past, to futile results.[2]
                    The Time Travelers 1964 Scientists test a time-viewing device but the screen becomes a portal which they enter; then it disappears, stranding them in a future devastated by nuclear war. [3]
                    Journey to the Center of Time 1967 Remake of The Time Travelers[4]
                    Le 15 Mai 1969 A couple wakes up one sunny morning on the 15th of May. Apparently, they had the same nightmare and the day unfolds strangely. The next day, we are still on the 15th of May. Claire Denis's graduation work from La Fémis.[5]
                    The Girl Who Leapt Through Time 1983 Based on Yasutaka Tsutsui's 1965 novel of the same name. The story is about a Japanese schoolgirl who accidentally gains the ability to time travel, which she experiments with and attempts to alter past events, leading her on a journey through multiple time loops.[6][7]
                    Urusei Yatsura 2: Beautiful Dreamer 1984 The students of Tomobiki high school relive the day before the school festival over and over.[8][9][10]
                    Mirror for a Hero 1987 Zerkalo dlya geroya, a Soviet Union film by Vladimir Khotinenko. Two heroes circle multiple times in 1949, adapting to the harsh post-war life, meeting the parents of one of them, finally returning to the present time, having reassessed their attitude towards elders.[11][12]
                    12:01 PM 1990 This short film is the first film adaptation of the short story "12:01 PM" by Richard A. Lupoff, which was published in 1973 in The Magazine of Fantasy and Science Fiction. It originally aired on cable television as part of the Showtime 30-Minute Movie anthology series. It was nominated for an Academy Award.[13]
                    12:01 1993 The second film adaptation of the short story "12:01 PM" by Richard A. Lupoff, which was published in 1973 in The Magazine of Fantasy and Science Fiction. Office worker Barry Thomas is forced to relive the worst day of his life.[14]
                    Groundhog Day 1993 Self-centered television weatherman Phil Connors (Bill Murray) is forced to relive the same day over and over.[15] This movie is generally seen as the quintessential time-loop movie by many with its name being synonymous with the genre as a whole.[16][17][18]
                    Christmas Every Day 1996 An American television movie based on William Dean Howells' 1892 short story "Christmas Every Day". A selfish teenager is forced to relive the same Christmas every day.[19]
                    Retroactive 1997 A psychiatrist makes multiple trips through time to save a woman who was murdered by her husband.[20]
                    Run Lola Run 1998 The story follows a woman named Lola (Franka Potente) who needs to obtain 100,000 Deutschmarks in twenty minutes to save the life of her boyfriend by resetting time multiple times by 20 minutes.[21]
                    Mickey's Once Upon a Christmas 1999 Huey, Dewey, and Louie must repeat Christmas Day until they come to learn the true meaning of Christmas.[22]
                    Naken 2000 On the day of his wedding, Anders wakes up naked in an elevator. As he attempts to piece together the reason for his predicament, the day repeats itself, and he becomes stuck in a time loop.[23]
                    Camp Slaughter 2005 A group of present-day teenagers are sent back to 1981, and discover a summer camp which is stuck reliving the day a demented killer went on a rampage.[24]
                    Christmas Do-Over 2006 A father is stuck in a time loop when his son wishes that every day was Christmas day. A remake of Christmas Every Day from an adult perspective.
                    The Girl Who Leapt Through Time 2006 Anime based on Yasutaka Tsutsui's 1965 novel of the same name.
                    Salvage 2006 Claire encounters horror and murder after finishing a day working at a convenience store, not once but many times. An official selection of the 2006 Sundance Festival.[25]
                    The Last Day of Summer 2007 11-year-old Luke gets his wish that every day could be the last day of summer.[26]
                    Frequently Asked Questions About Time Travel 2009 Three social misfits navigate a time-travel conundrum originating from their local pub.[27]
                    Triangle 2009 A group of friends go on a boat trip and become stranded due to a storm. They discover a cruise ship which causes them to experience a series of repeating events.[28]
                    The Girl Who Leapt Through Time 2010 Third film based on Yasutaka Tsutsui's 1965 novel of the same name.
                    Repeaters 2010 A group of inmates at a rehabilitation facility are forced to repeat the same day over and over.[29]
                    Source Code 2011 U.S. Army Aviation pilot Captain Colter Stevens repeatedly experiences the last eight minutes of another person's life to identify the bomber in a terrorist attack in order to prevent a second, nuclear attack on Chicago.[30]
                    The Road 2011 A group of teens finally work up the fortitude to explore a rural road, that has all the warning signs of being a bad idea.[31]
                    12 Dates of Christmas 2011 Kate finds herself reliving Christmas Eve (including a blind date with a man named Miles) over and over. She must discover how to break the cycle – should she attempt to win back her ex-boyfriend Jack, should she pursue Miles, or something else?[32]
                    Puella Magi Madoka Magica: The Movie 2012 Homura Akemi has the ability to travel 30 days back in time, and has done so countless times. Each month, Homura fails in her attempts to save Madoka Kaname from destruction, and is forced to relive the tragedy of losing Madoka.[33]
                    Mine Games 2012 A group of friends vacationing in the woods discover their own corpses and realize they are trapped in a time loop that they attempt to break.[34]
                    Blood Punch 2013 A trio visits a remote cabin for a one-time criminal endeavor and finds each member independently reliving the same day.[35]
                    Haunter 2013 The ghost of a teenager, Lisa Johnson (Abigail Breslin), who, along with her family, keeps reliving the same day (although she is the only one aware of it) tries to protect a young girl, Olivia (Eleanor Zichy), and her family from a dead serial killer, the Pale Man (Stephen McHattie), who can possess the living.[36]
                    El Ascensor 2014 On their fifth wedding anniversary, Sitio and Ana start arguing inside an elevator that keeps opening in their same floor. Emotionally and physically trapped, the two will have to work together to find a way out.[37]
                    One-Minute Time Machine 2014 This 6-minute comedic short film directed by Devon Avery and featuring Brian Dietzen and Erinn Hayes, parodies the time loop genre (and itself) when repeated applications of the device are used to redo a botched romantic approach.[38]
                    Oru Kanniyum Moonu Kalavaanikalum 2014 A group of friends find themselves in a time loop with 3 different time slots for the same incident and the result will vary due to the time and the result of people mood.[39]
                    Premature 2014 Rob, a high school senior, has to relive losing his virginity over and over until he gets it right, with the right girl.[40]
                    The Incident 2014 Two stories of groups trapped in infinite time and space loops. One, an endless staircase, the other, an endless road, both looping back on themselves. While time resets in their surroundings each day, the trapped characters continue to age within the loops.[41]
                    The Infinite Man 2014 After a romantic weekend break with his girlfriend goes awry, a scientist invents time travel in order to create for her the perfect getaway, but inadvertently traps her in a recurring temporal loop.[42]
                    Edge of Tomorrow 2014 Maj. William "Bill" Cage (Tom Cruise) and Special Forces soldier Rita Vrataski (Emily Blunt) team up to fight a hostile alien race known as Mimics, with Cage continually returning to a repeating battle through a time loop.[43]
                    Magnetic 2015 Alice is stuck in a time loop, repeating each week, trying to find a way to prevent the Earth from being destroyed by a massive solar flare.[44]
                    The Final Girls 2015 A group of friends find themselves trapped in an '80s horror movie that restarts every 92 minutes.[45]
                    ARQ 2016 An engineer, whose invention causes time to loop during a home invasion, attempts to save his former lover while learning who has targeted him and why.[46]
                    Doctor Strange 2016 Aided by the powers of The Eye of Agamotto, Stephen Strange traps himself and Dormammu in a time loop in order to bargain against him consuming the Earth.[47]
                    Erased 2016 A Japanese young man travels back in time to prevent the murder of his mother and others, looping through different versions of reality as his actions change the past.[48][49] (The same basic story exists in manga, anime, live action movie, and live action TV series versions.)
                    Groundhog Day for a Black Man 2016 Short film about a black man who keeps reliving the same day in which he ends up being shot by a police officer.[50] The director of this film accused Netflix's Two Distant Strangers of plagiarism.[51]
                    Miss Peregrine's Home for Peculiar Children 2016 Teenager Jake visits the ruins of an orphanage that was destroyed by a bombing raid during World War II and discovers that the original inhabitants survived by creating a time loop set to repeat the day of September 3, 1943.[52]
                    Naked 2017 Rob is caught in a time loop as he keeps waking up naked in a hotel elevator on the day of his wedding.[23] This is a remake of the 2000 Swedish film Naken.
                    Happy Death Day 2017 A college student is murdered on her birthday, Monday the 18th. She wakes up the morning of the 18th, alive, facing the same day over and over, while also getting weaker. She must figure out who her killer is in order to escape the loop.[53]
                    Before I Fall 2017 A popular high school senior finds herself reliving the same day over and over.[54]
                    The Endless 2017 Two brothers return to a cult that is stuck in a time loop.[55]
                    A Day 2017 A father attempts to save his daughter whilst stuck in an endless time loop.[56]
                    Reset 2017 A physicist uses an experimental wormhole machine to travel to parallel universes and back in time by an hour, trying to save her kidnapped son.[57]
                    The Fare 2018 A taxi driver and his passenger find themselves locked in an endless time loop as they have to repeat their journey over and over.[58]
                    When We First Met 2018 A guy’s grief over losing Ms. Right, grief he recovers from by time traveling to earn that magic “reset.” [59]
                    Koko-di Koko-da 2019 A struggling couple on a camping trip is caught in a time loop where they are tormented by a group of fairy-tale characters.[60]
                    Game Over 2019 Swapna, a wheelchair user, is defending her home from a mysterious intruder. Every time she is killed by the intruder, she awakens to the start of the day. However, the concept is more along the lines of how a video game works.[61]
                    The Obituary of Tunde Johnson 2019 A gay African-American teenager repeatedly relives the day of his fatal shooting in an unprovoked police brutality incident.[62]
                    Happy Death Day 2U 2019 Immediately after the events of the first film (Happy Death Day), Tree Gelbman (Jessica Rothe) unexpectedly re-enters the time loop except this time in an alternate reality.[63][64]
                    See You Yesterday 2019 An ambitious science prodigy uses her prowess and capabilities to create time machines, in order to save her brother who has been killed by a police officer. As she tries to alter the events of the past, she will eventually face the perilous consequences of time travel.[65]
                    Volition 2019 A man afflicted with clairvoyance tries to change his fate when a series of events leads to a vision of his own imminent murder.[66]
                    Two Distant Strangers 2020 An American short science-fiction film that examines the deaths of black Americans during encounters with police through the eyes of a character trapped in a time loop that keeps ending in his death.[67]
                    Palm Springs 2020 When carefree Nyles and reluctant maid of honor Sarah have a chance encounter at a Palm Springs wedding, things get complicated the next morning when they find themselves unable to escape the venue, themselves, or each other.[68]
                    Boss Level 2021 A retired special forces officer is trapped in a time loop on the day of his death.[69]
                    Christmas...Again?! 2021 A holiday Disney Channel Original Movie starring Scarlett Estevez as Ro, whose Christmas wish goes awry as she finds herself living christmas day over and over again.[70]
                    Churuli 2021 The film, directed by Lijo Jose Pellissery, tells the story of two undercover police officers who go in search of a wanted criminal named Mayiladumparambil Joy. However, they get stuck in the village of Churuli as eerie mysteries unfold.

                    The original story and characters are based on a short story from the book Kaligaminarile Kuttavalikal written by Vinoy Thomas.[71][72]

                    Jango 2021 A neurosurgeon who is stuck in a time loop tries to save his estranged wife from being killed by a mystery man.[73]
                    Maanaadu 2021 A man who is stuck in a time loop tries to stop a political rally from taking place in order to save the state's chief minister and prevent religious violence.[74]
                    The Map of Tiny Perfect Things 2021 Two teens live the same day repeatedly, enabling them to create the titular map.[75]
                    Meet Cute 2022 A Manhattan-native, Sheila, discovers a time machine in a nail salon and uses it to continually fix elements of a date she had the previous night.[76]
                    Mondays: see you "this" week 2022 The staff of an advertising agency office are stuck in a time loop except their boss.[77]
                    Vandits 2022 A group of incompetent criminals tries to rob a bingo hall, only to get trapped in a time loop in which they repeatedly stage the heist but fail in a different way each time.[78]
                    Zanox 2022 Misi, a high school student, relives his graduation day after he discovers his experimental medication has a weird side effect.[79]
                    Dreadful Chapters 2023 Indian Malayalam-language horror film directed by Nirmal Baby Varghese.[80]

                    See also[edit]

                    References[edit]

                    1. ^ Matthew Jones; Joan Ormrod (2015), Time Travel in Popular Media, McFarland & Company, ISBN 9780786478071
                    2. ^ "In the Loop: Repeat Performance"
                      . 24 August 2021.
                    3. ^ Bogue, Mike (September 26, 2017). Apocalypse Then: American and Japanese Atomic Cinema, 1951-1967. McFarland & Company. p. 80. ISBN 978-1476668413.
                    4. ^ Bogue, Mike (September 26, 2017). Apocalypse Then: American and Japanese Atomic Cinema, 1951-1967. McFarland & Company. p. 80. ISBN 978-1476668413.
                    5. ^ "Le 15 Mai"
                      .
                    6. ^ "THE GIRL WHO LEAPT THROUGH TIME (2006)"
                      . Deptford Cinema. August 9, 2017. Retrieved 27 January 2020.
                    7. ^ "THE GIRL WHO LEAPT THROUGH TIME (2006) at Deptford Cinema"
                      . TicketSource. 9 August 2017. Retrieved 27 January 2020.
                    8. ^ Hartl, John (September 3, 1998). "Disney Will Distribute Animated Japanese Movie"
                      . The Seattle Times. Retrieved May 15, 2017.
                    9. ^ French, Todd (July 1996). "Reviews"
                      . Cinefantastique. 27 (11/12): 123. Retrieved April 28, 2017. The student body of Tomobiki High experiences a major cause of deja vu when they find themselves endlessly repeating "the day before the School Festival."
                    10. ^ "Oshii says dissatisfaction behind 'eternal youth'". Yomiuri Shimbun. August 1, 2008. p. 18. The sequel, 1984's Urusei Yatsura 2: Beautiful Dreamer, meanwhile, included a gut-wrenching philosophical aspect in the storyline. In the movie, the characters relive the night before a school festival over and over, eventually blurring the line between reality and fantasy. The concept and execution took that film beyond mere anime geared toward kids.
                    11. ^ Bratersky, Alexander (29 July 2013). "'Alterum' Dance Troupe to Conquer Middle East"
                      . The Moscow Times. Retrieved 1 May 2015.
                    12. ^ Keller, Bill; Times, Special to the New York (23 April 1988). "A Movie Tribute for Stalin Generation"
                      . The New York Times. Retrieved 1 May 2015.
                    13. ^ "1990 (63rd): Short Film (Live Action)"
                      . Academy Awards Database. Academy of Motion Picture Arts and Sciences. Archived from the original
                      on 2015-04-02.
                    14. ^ "It's like Groundhog Day, but different. What is it?"
                      . TV Guide. Archived from the original
                      on 2014-03-28. Retrieved 2014-03-27.
                    15. ^ Fischer, Russ (2011-02-02). "'Groundhog Day:' Just How Long Was Phil Connors Stuck in a Time Loop?"
                      . Slashfilm. Retrieved 2014-03-27.
                    16. ^ "Over and over: why is Hollywood still obsessed with Groundhog Day?"
                      . the Guardian. 2019-02-13. Retrieved 2022-03-23.
                    17. ^ "Groundhog Day: the perfect comedy, for ever"
                      . the Guardian. 2013-02-07. Retrieved 2022-03-23.
                    18. ^ "The Pop Culture Legacy of 'Groundhog Day'"
                      . www.vice.com. 2 February 2017. Retrieved 2022-03-23.
                    19. ^ Caro, Jason. "Christmas Every Day"
                      . Radio Times. Retrieved 6 May 2014.
                    20. ^ Bailey, Keith. "Retroactive"
                      . Radio Times. Retrieved 2015-11-23.
                    21. ^ Cosmicneed, Eden (22 June 2021). "In the Loop: Run Lola Run"
                      . Cosmic Need. Retrieved 19 April 2023.
                    22. ^ Rain Fordem, Kajsa (24 December 2015). "Rotoscopers' 12 Days of Christmas: 'The Polar Express'"
                      . Rotoscopers. Retrieved 12 September 2017.
                    23. ^ Jump up to: a b Fleming, Mike Jr. (21 September 2016). "Netflix Gets 'Naked' With Marlon Wayans & Regina Hall"
                      . Deadline Hollywood. Retrieved 18 May 2017.
                    24. ^ Wheeler, Jeremy (2014). "Camp Slaughter (2005)"
                      . Movies & TV Dept. The New York Times. Archived from the original
                      on 12 May 2014. Retrieved 12 May 2014.
                    25. ^ Chang, Justin (31 January 2006). "Review: 'Salvage'"
                      . Variety. Retrieved 6 May 2014.
                    26. ^ Gutierrez, Reuben (2007). "The Last Day of Summer DVD Review"
                      . DVDizzy.com. Retrieved 6 May 2014.
                    27. ^ "Frequently Asked Questions About Time Travel (2009)"
                      . Rotten Tomatoes. Retrieved 2024-02-02.
                    28. ^ Li, Shirley (2009-10-01). "Triangle Review"
                      . Empire. Retrieved 2018-07-09.
                    29. ^ Alex Billington (2010-08-31). "Watch: Carl Bessai's Time Loop Sci-Fi Film 'Repeaters' Trailer"
                      . First Showing. Retrieved 2014-03-28.
                    30. ^ Lawler, Kelley (6 June 2014). "It's still 'Groundhog Day': The 4 best time-loop movies"
                      . USA Today. Retrieved 26 September 2017.
                    31. ^ Catsoulis, Jeannette. "Crawling Backwards In Time Into Haunted Real Estate"
                      . New York Times. Retrieved 2015-11-23.
                    32. ^ Griffiths, John (6 December 2011). "12 Dates of Christmas"
                      . US Weekly. Retrieved 7 May 2014.
                    33. ^ Loveridge, Lynzee (2013-08-10). "8 Essential Time-Travelling Heroines"
                      . Anime News Network.
                    34. ^ Bellete, Kwenton (17 August 2012). "MIFF 2012 Review: MINE GAMES world premiere"
                      . Screen Anarchy. Retrieved 26 September 2017.
                    35. ^ "Blood Punch - Fruity Looper"
                      . 9 March 2017. Retrieved 22 November 2018.
                    36. ^ Rotten, Doc. "Film Review: Haunter (2013)"
                      . HorrorNews. Retrieved 23 May 2014.
                    37. ^ Time Loop Movies – A Definitive 2023 List (Live The Same Day Over)
                    38. ^ Morgan, Nigel (May 12, 2020). "Review: One-Minute Time Machine"
                      . Retrieved August 17, 2023.
                    39. ^ "Movie Review : Oru Kanniyum Moonu Kalavaanikalum"
                      . Sify. Archived from the original
                      on 2014-04-06. Retrieved 2014-04-15.
                    40. ^ Leydon, Joe (2 July 2014). "Film Review: 'Premature'"
                      . Variety. Retrieved 31 August 2014.
                    41. ^ Film Review: ‘The Incident’
                    42. ^ Hedone. "Hedone / The Infinite Man"
                      . Retrieved 24 Feb 2018.
                    43. ^ Gettell, Oliver (2014-03-25). "'Edge of Tomorrow' trailer: Tom Cruise fights an eternal battle"
                      . Los Angeles Times. Retrieved 2014-03-28.
                    44. ^ "Another Hole In The Head Film Festival Reviews - Beyond Chron"
                      . beyondchron.org. 5 November 2015. Retrieved May 24, 2018.
                    45. ^ Harvey, Dennis (March 15, 2015). "SXSW Film Review: 'The Final Girls'"
                      . Variety.
                    46. ^ Tsoumbakos, Rachel (2016-09-01). "The Latest Netflix Original Movie 'ARQ' Is A Dystopian Time Loop Adventure"
                      . The Inquisitr. Retrieved 2016-10-26.
                    47. ^ Couch, Aaron (26 November 2016). "'Doctor Strange' Writer on Time-Bending Twist and What It Means for Marvel's Future"
                      . The Hollywood Reporter. Retrieved 21 April 2018.
                    48. ^ "ERASED"
                      . Anime News Network. January 8, 2016. Retrieved February 14, 2016.
                    49. ^ "僕だけがいない街"
                      . eiga.com. Retrieved 5 May 2022.
                    50. ^ Costello, Colin (2017-10-20). "'Groundhog Day for a Black Man' wows fest crowd"
                      . Reel Chicago. Retrieved 2021-06-16.
                    51. ^ Frias, Lauren (2021-05-06). "An Oscar-winning filmmaker responded to accusations that his short copied a 2016 film about police brutality: 'The concept of the time loop film device is not new'"
                      . Insider. Retrieved 2021-06-16.
                    52. ^ Hoffman, Jordan (September 25, 2016). "Miss Peregrine's Home For Peculiar Children review: mordant British YA X-Men is Tim Burton's best in 20 years"
                      . The Guardian.
                    53. ^ Hall, Jacob (15 June 2017). "'Happy Death Day' Trailer: 'Groundhog Day' With a Lot More Murder"
                      . /Film. Retrieved 16 June 2017.
                    54. ^ Felperin, Leslie (23 January 2017). "'Before I Fall': Film Review"
                      . The Hollywood Reporter. Retrieved 15 February 2017.
                    55. ^ Crucchiola, Jordan (April 6, 2018). "Unpacking That Big Twist in The Endless"
                      . Vulture.com. Retrieved February 19, 2020.
                    56. ^ Lee, Hyo-won (7 July 2017). "Cannes: South Korean Time Travel Film 'A Day' Sells to Multiple Territories"
                      . The Hollywood Reporter. Retrieved 25 September 2017.
                    57. ^ "'Reset' ('Ni Shi Ying Jiu'): Film Review"
                      . The Hollywood Reporter. 15 June 2017. Retrieved 3 March 2019.
                    58. ^ Harvey, Rupert (22 November 2019). "Movie Review: The Fare (2019)"
                      . The Critical Movie Critics. Retrieved 22 December 2019.
                    59. ^ Moore, Roger. "Netflixable? Can this couple recapture the magic of "When We First Met?""
                      . Movie Nation. Retrieved 19 June 2018.
                    60. ^ Aguilar, Carlos (February 4, 2019). "Time Loop Nightmare 'Koko-di Koko-da' Haunts A Couple In Crisis [Sundance Review]"
                      . The Playlist. Retrieved November 8, 2020.
                    61. ^ Lakshmi, Venkateswaran (June 12, 2019). "Game Over Movie Review"
                      . The Times of India. Retrieved October 19, 2020.
                    62. ^ Beandrea July, "'The Obituary of Tunde Johnson': Film Review | TIFF 2019"
                      . The Hollywood Reporter, September 8, 2019.
                    63. ^ Ouelette, Jennifer (1 December 2018). "A Pesky Time Loop Meets the Multiverse in Happy Death Day 2U First Trailer"
                      . /Film. Retrieved 1 December 2018.
                    64. ^ "Happy Death Day 2U Review"
                      . ScreenRant. 2019-02-13. Retrieved 2020-10-20.
                    65. ^ "'See You Yesterday' Review: An All-Too-Real Time-Travel Fantasy"
                      . The New York Times. 2019-05-17. Retrieved 2020-11-19.
                    66. ^ Debruge, Peter (July 8, 2020). "'Palm Springs': Film Review"
                      . Hollywood Reporter. Retrieved Jul 8, 2020.
                    67. ^ "Two Distant Strangers' Makes Sure Oscar Voters Don't Forget George Floyd"
                      . 2021-03-08. Retrieved 2021-04-15.
                    68. ^ Debruge, Peter (January 28, 2020). "'Palm Springs': Film Review"
                      . Variety. Retrieved Jul 3, 2020.
                    69. ^ Wiseman, Andreas (November 23, 2020). "Hulu Strikes U.S. Deal For Frank Grillo Action Sci-Fi Movie 'Boss Level'"
                      . Deadline. Retrieved November 26, 2020.
                    70. ^ "Disney+ Release Date Revealed For Disney Channel Original Movie "Christmas Again""
                      . Disney Plus Informer. 28 October 2021. Retrieved 4 January 2022.
                    71. ^ "CHURULI aka LABYRINTH Explanation"
                      . 20 November 2021.
                    72. ^ "Churuli and other Films based on Time Loops (Malayalam/Mathrubhumi Report)"
                      .
                    73. ^ "Jango review: Watch it for the time loop angle"
                      . Sify. Archived from the original
                      on November 19, 2021. Retrieved 19 November 2021.
                    74. ^ "Maanaadu Movie Review: A riveting take on time loop underlined by clever writing"
                      . Cinema Express. Retrieved 2021-11-25.
                    75. ^ Kenigsberg, Ben (2021-02-11). "'The Map of Tiny Perfect Things' Review: Another Do-Over"
                      . The New York Times. Retrieved 2021-02-12.
                    76. ^ D'Alessandro, Anthony (16 August 2022). "Kaley Cuoco & Pete Davidson Romantic Comedy 'Meet Cute' Sets Fall Premiere Date On Peacock"
                      . Deadline. Retrieved 30 August 2022.
                    77. ^ Hadfield, James (October 6, 2022). "'Mondays: See You "This" Week!': Zany office comedy gets stuck on repeat"
                      . The Japan Times. Retrieved October 30, 2022.
                    78. ^ John Kirk, "Vandits: Wacky Canuck Heist Film Vulgar, Lowbrow… and Awesome"
                      . Original Cin, December 14, 2022.
                    79. ^ "Criticism of Zanox - Risks and Side Effects - Fresno 24 English"
                      . fresno24.com. 2022-05-22. Retrieved 2022-06-20.
                    80. ^ "Dreadful Chapters: Nirmal Varghese unveils first-look poster of his horror film"
                      . Madhyamam. 19 July 2023. Retrieved 20 July 2022.
                    ================================================ FILE: test/test-pages/wordpress/expected-metadata.json ================================================ { "title": "Stack Overflow Jobs Data Shows ReactJS Skills in High Demand, WordPress Market Oversaturated with Developers", "byline": null, "dir": "ltr", "lang": "en-US", "excerpt": "Stack Overflow published its analysis of 2017 hiring trends based on the targeting options employers selected when posting to Stack Overflow Jobs. The report, which compares data from 200 companies…", "siteName": "WordPress Tavern", "publishedTime": "2017-03-09T23:16:02+00:00", "readerable": true } ================================================ FILE: test/test-pages/wordpress/expected.html ================================================

                    Stack Overflow published its analysis of 2017 hiring trends based on the targeting options employers selected when posting to Stack Overflow Jobs. The report, which compares data from 200 companies since 2015, ranks ReactJS, Docker, and Ansible at the top of the fastest growing skills in demand. When comparing the percentage change from 2015 to 2016, technologies like AJAX, Backbone.js, jQuery, and WordPress are less in demand.

                    Stack Overflow also measured the demand relative to the available developers in different tech skills. The demand for backend, mobile, and database engineers is higher than the number of qualified candidates available. WordPress is last among the oversaturated fields with a surplus of developers relative to available positions.

                    In looking at these results, it’s important to consider the inherent biases within the Stack Overflow ecosystem. In 2016, the site surveyed more than 56,000 developers but noted that the survey was “biased against devs who don’t speak English.” The average age of respondents was 29.6 years old and 92.8% of them were male.

                    For two years running, Stack Overflow survey respondents have ranked WordPress among the most dreaded technologies that they would prefer not to use. This may be one reason why employers wouldn’t be looking to advertise positions on the site’s job board, which is the primary source of the data for this report.

                    Many IT career forecasts focus more generally on job descriptions and highest paying positions. Stack Overflow is somewhat unique in that it identifies trends in specific tech skills, pulling this data out of how employers are tagging their listings for positions. It presents demand in terms of number of skilled developers relative to available positions, a slightly more complicated approach than measuring demand based on advertised salary. However, Stack Overflow’s data presentation could use some refining.

                    One commenter, Bruce Van Horn, noted that jobs tagged as “Full Stack Developer” already assume many of the skills that are listed separately:

                    I wonder how many of these skills are no longer listed because they are “table stakes”. You used to have to put CSS, jQuery, and JSON on the job description. I wouldn’t expect to have to put that on a Full Stack Developer description today – if you don’t know those then you aren’t a Full Stack Web Developer, and I’m more interested in whether you know the shiny things like React, Redux, and Angular2.

                    It would be interesting to know what is meant by tagging “WordPress” as a skill – whether it is the general ability to work within the WordPress ecosystem of tools or if it refers to specific skills like PHP. Browsing a few jobs on Stack Overflow, WordPress positions vary in the skills they require, such as React.js, Angular, PHP, HTML, CSS, and other technologies. This is a reflection of the diversity of technology that can be leveraged in creating WordPress-powered sites and applications, and several of these skills are listed independently of WordPress in the data.

                    Regardless of how much credibility you give Stack Overflow’s analysis of hiring trends, the report’s recommendation for those working in technologies oversaturated with developers is a good one: “Consider brushing up on some technologies that offer higher employer demand and less competition.” WordPress’ code base is currently 59% PHP and 27% JavaScript. The percentage of PHP has grown over time, but newer features and improvements to core are also being built in JavaScript. These are both highly portable skills that are in demand on the web.

                    ================================================ FILE: test/test-pages/wordpress/source.html ================================================ Stack Overflow Jobs Data Shows ReactJS Skills in High Demand, WordPress Market Oversaturated with Developers – WordPress Tavern

                    Stack Overflow Jobs Data Shows ReactJS Skills in High Demand, WordPress Market Oversaturated with Developers

                    13

                    Stack Overflow published its analysis of 2017 hiring trends based on the targeting options employers selected when posting to Stack Overflow Jobs. The report, which compares data from 200 companies since 2015, ranks ReactJS, Docker, and Ansible at the top of the fastest growing skills in demand. When comparing the percentage change from 2015 to 2016, technologies like AJAX, Backbone.js, jQuery, and WordPress are less in demand.

                    Stack Overflow also measured the demand relative to the available developers in different tech skills. The demand for backend, mobile, and database engineers is higher than the number of qualified candidates available. WordPress is last among the oversaturated fields with a surplus of developers relative to available positions.

                    In looking at these results, it’s important to consider the inherent biases within the Stack Overflow ecosystem. In 2016, the site surveyed more than 56,000 developers but noted that the survey was “biased against devs who don’t speak English.” The average age of respondents was 29.6 years old and 92.8% of them were male.

                    For two years running, Stack Overflow survey respondents have ranked WordPress among the most dreaded technologies that they would prefer not to use. This may be one reason why employers wouldn’t be looking to advertise positions on the site’s job board, which is the primary source of the data for this report.

                    Many IT career forecasts focus more generally on job descriptions and highest paying positions. Stack Overflow is somewhat unique in that it identifies trends in specific tech skills, pulling this data out of how employers are tagging their listings for positions. It presents demand in terms of number of skilled developers relative to available positions, a slightly more complicated approach than measuring demand based on advertised salary. However, Stack Overflow’s data presentation could use some refining.

                    One commenter, Bruce Van Horn, noted that jobs tagged as “Full Stack Developer” already assume many of the skills that are listed separately:

                    I wonder how many of these skills are no longer listed because they are “table stakes”. You used to have to put CSS, jQuery, and JSON on the job description. I wouldn’t expect to have to put that on a Full Stack Developer description today – if you don’t know those then you aren’t a Full Stack Web Developer, and I’m more interested in whether you know the shiny things like React, Redux, and Angular2.

                    It would be interesting to know what is meant by tagging “WordPress” as a skill – whether it is the general ability to work within the WordPress ecosystem of tools or if it refers to specific skills like PHP. Browsing a few jobs on Stack Overflow, WordPress positions vary in the skills they require, such as React.js, Angular, PHP, HTML, CSS, and other technologies. This is a reflection of the diversity of technology that can be leveraged in creating WordPress-powered sites and applications, and several of these skills are listed independently of WordPress in the data.

                    Regardless of how much credibility you give Stack Overflow’s analysis of hiring trends, the report’s recommendation for those working in technologies oversaturated with developers is a good one: “Consider brushing up on some technologies that offer higher employer demand and less competition.” WordPress’ code base is currently 59% PHP and 27% JavaScript. The percentage of PHP has grown over time, but newer features and improvements to core are also being built in JavaScript. These are both highly portable skills that are in demand on the web.

                    There are 13 comments

                    One nuance of over saturation number might be that many people who self–identify themselves as WordPress “developers” might be in “site builder” segment. Off–the–shelf assembly and lightweight customization rather than involved custom development.

                    I had certainly never had an impression that there is an oversupply of of WP devs with advanced level of PHP and experience in custom projects.Report

                    I feel like you’re 100% right on this one. WordPress has many self called ‘developers’ who actually don’t know how to code. I think there’s indeed a lot competition in that space (the “Off–the–shelf assembly and lightweight customization”) but there’s still more than enough work for developers that can actually code complex custom solutions in WP.Report

                    If I consider myself a ‘page builder’? If you’re asking if I do “Off–the–shelf assembly and lightweight customization” WordPress work, no I don’t. I make and sell WordPress plugins that allow others to do so though :)Report

                    You’re 100% correct Rarst. In my opinion, if a person does not have the skills of a computer scientist (OOP specifically) then they cannot really hold the title of a “developer,” albeit outside of project management skills; UML, scrum, etc. Developers can develop because when their calculator breaks, they can still do the math, so to speak. It will just be a little slower.

                    Installation, light-weight customization, tweaks, etc., relative to the WP theme design or not, is NOT true development. You must know code/syntax, algorithms (e.g. optimization techniques), etc. to truly “develop” big [project] picture, small picture, and the realtionships between them to tweak code. THAT is development.Report

                    Agree with you. WordPress is heading towards the right direction and It is necessary to implement REST API quickly to make it more accessible for people who hate to deal with PHP or WordPress tags. Report

                    I’m a “BuddyPress developer”, as well as quite a bunch of other people.
                    The problem is that BP is not that big market, it’s not that easy to be 100% focused on BP-related projects only. So such people are either working with WP (mainly) and BP (when something – seldom – appear), or have own business around WP (and sometimes) BP. And such people are marketing WP skills more, I guess.Report

                    Why doesn’t WordPress have some kind of certification system when it comes to working with WordPress? Make it something similar to CompTIA A+ certification. There could be different levels of verification to cover all aspects of WordPress. It would be a benefit to anyone trying to make a living working with WordPress. It would also be a major benefit for anyone seeking to hire a WordPress professional. There could be a centralized location on dot org that would list certified WordPress professionals tagged with their various certifications.

                    Currently, it’s so hit or miss when it comes to hiring someone to provide help with WordPress. You might need someone with just a basic skill set to set up a new site with a standard theme and some basic plugins, but the person you hire is more suited for advanced, complex integrations involving multiple APIs. The problem is, the customer ends up paying far more than they really need to for someone with skill sets far more advanced than what they actually need. Report

                    In theory, this might be a good idea, though just who would do the certifying? In practice, however, I’ve heard from people who’ve been developing compiled software since before PHP existed that a lot of those certifications really test your ability to take standardized tests, not your ability as a programmer. The best way to hire anybody to do anything is to get a personal recommendation.Report

                    WordPress.org should do the certifications. Individual testing could happen at WordCamps and/or at local testing facilities like how CompTIA A+ certification is handled. Most clients do not need a programmer or a developer. They need someone who can set up a website. That means finding a host, installing WordPress, installing a theme, and installing plugins.

                    If someone hires a programmer to do all that, they are most likely overpaying for the work.

                    Report

                    Were you volunteering to help design the tests and conduct the certifications? Or at least set up the means to organize it on make.wordpress.org? It’s an idea that has potential merit and you aren’t the first person to wish there were some kind of standards by which to judge a person’s basic WordPress literacy.

                    To the best of my knowledge, however, everything done by WordPress.org is handled by volunteers. In the spirit of open source, if there’s something you want to see happen, you need to get it started.

                    An idea like certification would actually need much broader support throughout the WordPress community than a choice of what to focus development on for the next release. And it would require either a means to administer and grade the test online automatically (requiring programming to make it work) or a fairly large army of volunteers to undertake in-person testing. I imagine you could tie up years of surveys and committee meetings just figuring out what you wanted to test for, never mind designing the test itself.

                    And you have to decide how often people need to re-certify, because recommendations and best practices change rapidly in the web world.

                    Even if you get that far, it will only make a difference if enough people accept the validity of the test and care whether someone has passed it before hiring them.

                    And, finally, a test like that could determine whether the person had enough skill to install WordPress, set up a theme, and choose some plugins, or (at a more advanced level) whether they understand WordPress’ action and filter hooks, plus PHP, HTML, CSS, and JavaScript, but it will never tell you whether a person is honest, timely, able to understand (or even ask) what a client’s real needs and goals are, or someone you can work with without both of you going crazy.Report

                    Your email address will not be published. Required fields are marked *

                    :)

                    Original text

                    Contribute a better translation
                    ================================================ FILE: test/test-pages/yahoo-1/expected-metadata.json ================================================ { "title": "These are the 8 coolest PlayStation VR games", "byline": "Ben Silverman", "dir": null, "lang": "en-US", "excerpt": "To help you decide what’s what, I’ve put together this list of the 8 PSVR games worth considering. Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/yahoo-1/expected.html ================================================

                    The PlayStation VR

                    Sony’s PlayStation VR.

                    Virtual reality has officially reached the consoles. And it’s pretty good! Sony’s PlayStation VR is extremely comfortable and reasonably priced, and while it’s lacking killer apps, it’s loaded with lots of interesting ones.

                    But which ones should you buy? I’ve played just about every launch game, and while some are worth your time, others you might want to skip. To help you decide what’s what, I’ve put together this list of the eight PSVR games worth considering.

                    “Rez Infinite” ($30)

                    Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. It includes a fully remastered take on the original “Rez” – you zoom through a Matrix-like computer system, shooting down enemies to the steady beat of thumping electronica – but the VR setting makes it incredibly immersive. It gets better the more you play it, too; unlock the amazing Area X mode and you’ll find yourself flying, shooting and bobbing your head to some of the trippiest visuals yet seen in VR.

                    “Thumper” ($20)

                    What would happen if Tron, the board game Simon, a Clown beetle, Cthulhu and a noise band met in VR? Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well, a violent rhythm game that’s also a gorgeous, unsettling and totally captivating assault on the senses. With simple controls and a straightforward premise – click the X button and the analog stick in time with the music as you barrel down a neon highway — it’s one of the rare games that works equally well both in and out of VR. But since you have PSVR, play it there. It’s marvelous.

                    “Until Dawn: Rush of Blood” ($20)

                    Cheeky horror game “Until Dawn” was a breakout hit for the PS4 last year, channeling the classic “dumb teens in the woods” horror trope into an effective interactive drama. Well, forget all that if you fire up “Rush of Blood,” because this one sticks you front and center on a rollercoaster ride from Hell. Literally. You ride through a dimly-lit carnival of terror, dual-wielding pistols as you take down targets, hideous pig monsters and, naturally, maniac clowns. Be warned: If the bad guys don’t get you, the jump scares will.

                    “Headmaster” ($20)

                    Soccer meets “Portal” in the weird (and weirdly fun) “Headmaster,” a game about heading soccer balls into nets, targets and a variety of other things while stuck in some diabolical training facility. While at first it seems a little basic, increasingly challenging shots and a consistently entertaining narrative keep it from running off the pitch. Funny, ridiculous and as easy as literally moving your head back and forth, it’s a pleasant PSVR surprise.

                    “RIGS: Mechanized Combat League” ($50)

                    Giant mechs + sports? That’s the gist of this robotic blast-a-thon, which pits two teams of three against one another in gorgeous, explosive and downright fun VR combat. At its best, “RIGS” marries the thrill of fast-paced competitive shooters with the insanity of piloting a giant mech in VR. It can, however, be one of the barfier PSVR games. So pack your Dramamine, you’re going to have to ease yourself into this one.

                    “Batman Arkham VR” ($20)

                    “I’m Batman,” you will say. And you’ll actually be right this time, because you are Batman in this detective yarn, and you know this because you actually grab the famous cowl and mask, stick it on your head, and stare into the mirrored reflection of Rocksteady Games’ impressive Dark Knight character model. It lacks the action of its fellow “Arkham” games and runs disappointingly short, but it’s a high-quality experience that really shows off how powerfully immersive VR can be.

                    “Job Simulator” ($30)

                    There are a number of good VR ports in the PSVR launch lineup, but the HTC Vive launch game “Job Simulator” might be the best. Your task? Lots of tasks, actually, from cooking food to fixing cars to working in an office, all for robots, because did I mention you were in the future? Infinitely charming and surprisingly challenging, it’s a great showpiece for VR.

                    “Eve Valkyrie” ($60)

                    Already a hit on the Oculus Rift, this space dogfighting game was one of the first to really show off how VR can turn a traditional game experience into something special. It’s pricey and not quite as hi-res as the Rift version, but “Eve Valkyrie” does an admirable job filling the void left since “Battlestar Galactica” ended. Too bad there aren’t any Cylons in it (or are there?)

                    More games news:

                    Ben Silverman is on Twitter at ben_silverman.

                    ================================================ FILE: test/test-pages/yahoo-1/source.html ================================================ These are the 8 coolest PlayStation VR games

                    These are the 8 coolest PlayStation VR games

                    Ben Silverman
                    Games Editor
                    Yahoo Finance
                    The PlayStation VR
                    Sony’s PlayStation VR.

                    Virtual reality has officially reached the consoles. And it’s pretty good! Sony’s PlayStation VR is extremely comfortable and reasonably priced, and while it’s lacking killer apps, it’s loaded with lots of interesting ones.

                    But which ones should you buy? I’ve played just about every launch game, and while some are worth your time, others you might want to skip. To help you decide what’s what, I’ve put together this list of the eight PSVR games worth considering.

                    “Rez Infinite” ($30)

                    Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. It includes a fully remastered take on the original “Rez” – you zoom through a Matrix-like computer system, shooting down enemies to the steady beat of thumping electronica – but the VR setting makes it incredibly immersive. It gets better the more you play it, too; unlock the amazing Area X mode and you’ll find yourself flying, shooting and bobbing your head to some of the trippiest visuals yet seen in VR.

                    “Thumper” ($20)

                    What would happen if Tron, the board game Simon, a Clown beetle, Cthulhu and a noise band met in VR? Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well, a violent rhythm game that’s also a gorgeous, unsettling and totally captivating assault on the senses. With simple controls and a straightforward premise – click the X button and the analog stick in time with the music as you barrel down a neon highway — it’s one of the rare games that works equally well both in and out of VR. But since you have PSVR, play it there. It’s marvelous.

                    “Until Dawn: Rush of Blood” ($20)

                    Cheeky horror game “Until Dawn” was a breakout hit for the PS4 last year, channeling the classic “dumb teens in the woods” horror trope into an effective interactive drama. Well, forget all that if you fire up “Rush of Blood,” because this one sticks you front and center on a rollercoaster ride from Hell. Literally. You ride through a dimly-lit carnival of terror, dual-wielding pistols as you take down targets, hideous pig monsters and, naturally, maniac clowns. Be warned: If the bad guys don’t get you, the jump scares will.

                    “Headmaster” ($20)

                    Soccer meets “Portal” in the weird (and weirdly fun) “Headmaster,” a game about heading soccer balls into nets, targets and a variety of other things while stuck in some diabolical training facility. While at first it seems a little basic, increasingly challenging shots and a consistently entertaining narrative keep it from running off the pitch. Funny, ridiculous and as easy as literally moving your head back and forth, it’s a pleasant PSVR surprise.

                    “RIGS: Mechanized Combat League” ($50)

                    Giant mechs + sports? That’s the gist of this robotic blast-a-thon, which pits two teams of three against one another in gorgeous, explosive and downright fun VR combat. At its best, “RIGS” marries the thrill of fast-paced competitive shooters with the insanity of piloting a giant mech in VR. It can, however, be one of the barfier PSVR games. So pack your Dramamine, you’re going to have to ease yourself into this one.

                    “Batman Arkham VR” ($20)

                    “I’m Batman,” you will say. And you’ll actually be right this time, because you are Batman in this detective yarn, and you know this because you actually grab the famous cowl and mask, stick it on your head, and stare into the mirrored reflection of Rocksteady Games’ impressive Dark Knight character model. It lacks the action of its fellow “Arkham” games and runs disappointingly short, but it’s a high-quality experience that really shows off how powerfully immersive VR can be.

                    “Job Simulator” ($30)

                    There are a number of good VR ports in the PSVR launch lineup, but the HTC Vive launch game “Job Simulator” might be the best. Your task? Lots of tasks, actually, from cooking food to fixing cars to working in an office, all for robots, because did I mention you were in the future? Infinitely charming and surprisingly challenging, it’s a great showpiece for VR.

                    “Eve Valkyrie” ($60)

                    Already a hit on the Oculus Rift, this space dogfighting game was one of the first to really show off how VR can turn a traditional game experience into something special. It’s pricey and not quite as hi-res as the Rift version, but “Eve Valkyrie” does an admirable job filling the void left since “Battlestar Galactica” ended. Too bad there aren’t any Cylons in it (or are there?)

                    More games news:

                    Ben Silverman is on Twitter at ben_silverman.

                    ================================================ FILE: test/test-pages/yahoo-2/expected-metadata.json ================================================ { "title": "Yahoo News - Latest News & Headlines", "byline": "NATALIYA VASILYEVA", "dir": null, "lang": "en-US", "excerpt": "The latest news and headlines from Yahoo! News. Get breaking news stories and in-depth coverage with videos and photos.", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/yahoo-2/expected.html ================================================

                    1 / 5

                    In this photo dated Tuesday, Nov, 29, 2016 the Soyuz-FG rocket booster with the Progress MS-04 cargo ship is installed on a launch pad in Baikonur, Kazakhstan. The unmanned Russian cargo space ship Progress MS-04 broke up in the atmosphere over Siberia on Thursday Dec. 1, 2016, just minutes after the launch en route to the International Space Station due to an unspecified malfunction, the Russian space agency said.(Oleg Urusov/ Roscosmos Space Agency Press Service photo via AP)

                    In this photo dated Tuesday, Nov, 29, 2016 the Soyuz-FG rocket booster with the Progress MS-04 cargo ship is installed on a launch pad in Baikonur, Kazakhstan. The unmanned Russian cargo space ship Progress MS-04 broke up in the atmosphere over Siberia on Thursday Dec. 1, 2016, just minutes after the launch en route to the International Space Station due to an unspecified malfunction, the Russian space agency said.(Oleg Urusov/ Roscosmos Space Agency Press Service photo via AP)

                    MOSCOW (AP) — An unmanned Russian cargo spaceship heading to the International Space Station broke up in the atmosphere over Siberia on Thursday due to an unspecified malfunction, the Russian space agency said.

                    The Progress MS-04 cargo craft broke up at an altitude of 190 kilometers (118 miles) over the remote Russian Tuva region in Siberia that borders Mongolia, Roscosmos said in a statement. It said most of spaceship's debris burnt up as it entered the atmosphere but some fell to Earth over what it called an uninhabited area.

                    Local people reported seeing a flash of light and hearing a loud thud west of the regional capital of Kyzyl, more than 3,600 kilometers (2,200 miles) east of Moscow, the Tuva government was quoted as saying late Thursday by the Interfax news agency.

                    The Progress cargo ship had lifted off as scheduled at 8:51 p.m. (1451 GMT) from Russia's space launch complex in Baikonur, Kazakhstan, to deliver 2.5 metric tons of fuel, water, food and other supplies. It was set to dock with the space station on Saturday.

                    Roscosmos said the craft was operating normally before it stopped transmitting data 6 ½ minutes after the launch. The Russian space agency would not immediately describe the malfunction, saying its experts were looking into it.

                    This is the third botched launch of a Russian spacecraft in two years. A Progress cargo ship plunged into the Pacific Ocean in May 2015, and a Proton-M rocket carrying an advanced satellite broke up in the atmosphere in May 2014.

                    But both Roscosmos and NASA said the crash of the ship would have no impact on the operations of the orbiting space lab that is currently home to a six-member crew, including three cosmonauts from Russia, two NASA astronauts and one from the European Union.

                    Orbital ATK, NASA's other shipper, successfully sent up supplies to the space station in October, and a Japanese cargo spaceship is scheduled to launch a full load in mid-December.

                    NASA supplier SpaceX, meanwhile, has been grounded since a rocket explosion in September on the launch pad at Cape Canaveral, Florida. The company hopes to resume launches in December to deliver communication satellites.

                    ___

                    This version corrects the spelling of the region to Tuva, not Tyva.

                    __

                    Aerospace Writer Marcia Dunn in Cape Canaveral, Florida, and Vladimir Isachenkov in Moscow contributed to this report.

                    ================================================ FILE: test/test-pages/yahoo-2/source.html ================================================ Russia: Space ship malfunctions, breaks up over Siberia

                    Russia: Space ship malfunctions, breaks up over Siberia

                    NATALIYA VASILYEVA
                    Associated Press
                    1 / 5

                    In this photo dated Tuesday, Nov, 29, 2016 the Soyuz-FG rocket booster with the Progress MS-04 cargo ship is installed on a launch pad in Baikonur, Kazakhstan. The unmanned Russian cargo space ship Progress MS-04 broke up in the atmosphere over Siberia on Thursday Dec. 1, 2016, just minutes after the launch en route to the International Space Station due to an unspecified malfunction, the Russian space agency said.(Oleg Urusov/ Roscosmos Space Agency Press Service photo via AP)

                    In this photo dated Tuesday, Nov, 29, 2016 the Soyuz-FG rocket booster with the Progress MS-04 cargo ship is installed on a launch pad in Baikonur, Kazakhstan. The unmanned Russian cargo space ship Progress MS-04 broke up in the atmosphere over Siberia on Thursday Dec. 1, 2016, just minutes after the launch en route to the International Space Station due to an unspecified malfunction, the Russian space agency said.(Oleg Urusov/ Roscosmos Space Agency Press Service photo via AP)

                    MOSCOW (AP) — An unmanned Russian cargo spaceship heading to the International Space Station broke up in the atmosphere over Siberia on Thursday due to an unspecified malfunction, the Russian space agency said.

                    The Progress MS-04 cargo craft broke up at an altitude of 190 kilometers (118 miles) over the remote Russian Tuva region in Siberia that borders Mongolia, Roscosmos said in a statement. It said most of spaceship's debris burnt up as it entered the atmosphere but some fell to Earth over what it called an uninhabited area.

                    Local people reported seeing a flash of light and hearing a loud thud west of the regional capital of Kyzyl, more than 3,600 kilometers (2,200 miles) east of Moscow, the Tuva government was quoted as saying late Thursday by the Interfax news agency.

                    The Progress cargo ship had lifted off as scheduled at 8:51 p.m. (1451 GMT) from Russia's space launch complex in Baikonur, Kazakhstan, to deliver 2.5 metric tons of fuel, water, food and other supplies. It was set to dock with the space station on Saturday.

                    Roscosmos said the craft was operating normally before it stopped transmitting data 6 ½ minutes after the launch. The Russian space agency would not immediately describe the malfunction, saying its experts were looking into it.

                    This is the third botched launch of a Russian spacecraft in two years. A Progress cargo ship plunged into the Pacific Ocean in May 2015, and a Proton-M rocket carrying an advanced satellite broke up in the atmosphere in May 2014.

                    But both Roscosmos and NASA said the crash of the ship would have no impact on the operations of the orbiting space lab that is currently home to a six-member crew, including three cosmonauts from Russia, two NASA astronauts and one from the European Union.

                    Orbital ATK, NASA's other shipper, successfully sent up supplies to the space station in October, and a Japanese cargo spaceship is scheduled to launch a full load in mid-December.

                    NASA supplier SpaceX, meanwhile, has been grounded since a rocket explosion in September on the launch pad at Cape Canaveral, Florida. The company hopes to resume launches in December to deliver communication satellites.

                    ___

                    This version corrects the spelling of the region to Tuva, not Tyva.

                    __

                    Aerospace Writer Marcia Dunn in Cape Canaveral, Florida, and Vladimir Isachenkov in Moscow contributed to this report.

                    ================================================ FILE: test/test-pages/yahoo-3/expected-metadata.json ================================================ { "title": "Veteran Wraps Baby in American Flag, Photo Sparks Controversy", "byline": "By GILLIAN MOHNEY\n March 11, 2015 3:46 PM", "dir": "ltr", "lang": "en-US", "excerpt": "A photographer and Navy veteran is fighting back after a photo she posted to Facebook started an online backlash. Vanessa Hicks said she had no idea her photo would be considered controversial. The photo, from a military family’s newborn photo shoot, showed a newborn infant wrapped in an American flag held by his father, who was in his military uniform. Hicks, a Navy veteran herself and the wife of an active-duty Navy member, said her intention was to honor the flag as well as her clients, who wanted to incorporate their military service in the photo shoot.", "siteName": "Yahoo", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/yahoo-3/expected.html ================================================

                    'GMA' Cookie Search:

                    A photographer and Navy veteran is fighting back after a photo she posted to Facebook started an online backlash.

                    Vanessa Hicks said she had no idea her photo would be considered controversial. The photo, from a military family’s newborn photo shoot, showed a newborn infant wrapped in an American flag held by his father, who was in his military uniform.

                    Hicks, a Navy veteran herself and the wife of an active-duty Navy member, said her intention was to honor the flag as well as her clients, who wanted to incorporate their military service in the photo shoot.

                    Pizza Man Making Special Delivery Pizza Delivery to Afghanistan During Super Bowl

                    Redesigned Scopes Fail to Stop 'Superbug Outbreaks

                    Antarctica 'Penguin Post Office' Attracts Record Number of Applicants

                    “This is what he was fighting for, his son wrapped in an American flag,” Hicks told ABC News. However, when she posted the image on her page, she started to get comments accusing her of desecrating the flag.

                    On one Facebook page an unidentified poster put up her picture writing and wrote they found it was “disrespectful, rude, tacky, disgusting, and against the U.S. Flag Code.”

                    View photo

                    .

                    Vanessa Hicks

                    Vanessa Hicks

                    The Federal Flag Code has guidelines for the proper treatment of the U.S. Flag but there are no rules for punishment related to violations. In the past, the Supreme Court has found that people are protected from punishment under the First Amendment for manipulating or even burning the flag.

                    Hicks said she was surprised when messages suddenly started to pop up on her Facebook page and even her own website criticizing her photos.

                    She said she stayed up until 4 a.m. recently to take down comments from her business and company page, even on shoots that had nothing to do with the flag.

                    “I know how low I felt during those first few hours,” said Hicks. “[I felt] am I not a good American or veteran or wife. It’s a train-wreck you can’t help but watch.”

                    As Hicks tried to stop the comments from taking over her pages, others started to take notice and her picture went viral on social media sites. After that, Hicks found that many people, both military and civilian, told her they did not find the picture offensive.

                    “I have seen first-hand what is desecration of the flag,” Hicks said of her time in the military. “At the end of the day I didn’t do anything that disrespected this flag.”

                    Hicks, whose husband is still on active duty in the Navy, said the flag is a symbol of U.S. freedoms including the First Amendment right to free speech.

                    “[My husband] wouldn’t die for a flag, he would die for the freedoms that this country offers,” she told ABC News.

                    After her story grabbed local headlines, Hicks has been inundated by requests for photos shoots, and she said she plans to give 15 percent of all profits related to these shoots to the USO.

                    ================================================ FILE: test/test-pages/yahoo-3/source.html ================================================
                    Veteran Wraps Baby in American Flag, Photo Sparks Controversy - Yahoo
                    'GMA' Cookie Search:
                    Share Your Recipe

                    Veteran Wraps Baby in American Flag, Photo Sparks Controversy

                    By GILLIAN MOHNEY March 11, 2015 3:46 PM Good Morning America
                    Content preferences

                      Captions Not Available
                      Settings
                      Full Screen
                      Embed
                      Video Quality
                      Auto
                      High (HD)
                      Medium
                      Low
                      Closed Caption Settings
                      Back
                      Closed Captions
                      Language
                      English
                      • No item available
                      Position
                      • Bottom
                      • Top
                      Type Size
                      A
                      A
                      A
                      A
                      A
                      Aa
                      Style Options
                      Captions will look like this
                      Back
                      • Presets
                      • Text
                      • background
                      • effects
                      Aa
                      Aa
                      Aa
                      Aa
                      Aa
                      Aa
                      Aa
                      Aa
                      Arial
                      • Georgia
                      • Palatino Linotype
                      • Times New Roman
                      • Arial
                      • Arial Black
                      • Comic Sans MS
                      • Impact
                      • Lucida Sans Unicode
                      • Tahoma
                      • Trebuchet MS
                      • Verdana
                      • Courier New
                      • Lucida Console
                      75% (transparency)
                      • 100% (opaque)
                      • 75% (transparency)
                      • 50% (transparency)
                      • 25% (transparency)

                      Aa

                      Plain

                      Aa

                      Shadow

                      Aa

                      raised

                      Aa

                      Lowered

                      Aa

                      Outline
                      Captions will look like this
                      Video Not Available
                      Unfortunately, this video is not available in your region.
                      SS-100-202

                      Photo of Baby Wrapped in US Flag Seen as Unpatriotic

                      Now watching

                      Next video starts in : 7 Play

                      Photo of Baby Wrapped in US Flag Seen as Unpatriotic

                      A photographer and Navy veteran is fighting back after a photo she posted to Facebook started an online backlash.

                      Vanessa Hicks said she had no idea her photo would be considered controversial. The photo, from a military family’s newborn photo shoot, showed a newborn infant wrapped in an American flag held by his father, who was in his military uniform.

                      Hicks, a Navy veteran herself and the wife of an active-duty Navy member, said her intention was to honor the flag as well as her clients, who wanted to incorporate their military service in the photo shoot.

                      Pizza Man Making Special Delivery Pizza Delivery to Afghanistan During Super Bowl

                      Redesigned Scopes Fail to Stop 'Superbug Outbreaks

                      Antarctica 'Penguin Post Office' Attracts Record Number of Applicants

                      “This is what he was fighting for, his son wrapped in an American flag,” Hicks told ABC News. However, when she posted the image on her page, she started to get comments accusing her of desecrating the flag.

                      On one Facebook page an unidentified poster put up her picture writing and wrote they found it was “disrespectful, rude, tacky, disgusting, and against the U.S. Flag Code.”

                      View photo

                      .
                      Vanessa Hicks

                      Vanessa Hicks
                      The Federal Flag Code has guidelines for the proper treatment of the U.S. Flag but there are no rules for punishment related to violations. In the past, the Supreme Court has found that people are protected from punishment under the First Amendment for manipulating or even burning the flag.

                      Hicks said she was surprised when messages suddenly started to pop up on her Facebook page and even her own website criticizing her photos.

                      She said she stayed up until 4 a.m. recently to take down comments from her business and company page, even on shoots that had nothing to do with the flag.

                      “I know how low I felt during those first few hours,” said Hicks. “[I felt] am I not a good American or veteran or wife. It’s a train-wreck you can’t help but watch.”

                      As Hicks tried to stop the comments from taking over her pages, others started to take notice and her picture went viral on social media sites. After that, Hicks found that many people, both military and civilian, told her they did not find the picture offensive.

                      “I have seen first-hand what is desecration of the flag,” Hicks said of her time in the military. “At the end of the day I didn’t do anything that disrespected this flag.”

                      Hicks, whose husband is still on active duty in the Navy, said the flag is a symbol of U.S. freedoms including the First Amendment right to free speech.

                      “[My husband] wouldn’t die for a flag, he would die for the freedoms that this country offers,” she told ABC News.

                      After her story grabbed local headlines, Hicks has been inundated by requests for photos shoots, and she said she plans to give 15 percent of all profits related to these shoots to the USO.

                      Sorry you didn't like this comment. Please provide a reason below.

                      Are you sure?
                      Rating failed. Try again.
                      Request failed. Try again.
                      We will promote constructive and witty comments to the top, so everyone sees them!
                      Sorry, we can’t load comments right now. Try again.

                        Recommended for You

                        ================================================ FILE: test/test-pages/yahoo-4/expected-metadata.json ================================================ { "title": "トレンドマイクロ、公衆無線LANを安全に使うためのアプリ「フリーWi-Fiプロテクション」（CNET Japan） - Yahoo!ニュース", "byline": "個人", "dir": null, "lang": "ja", "excerpt": "トレンドマイクロは3月9日、Wi-Fi利用時の通信を暗号化し保護するスマホ・タブレット - Yahoo!ニュース(CNET Japan)", "siteName": "Yahoo!ニュース", "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/yahoo-4/expected.html ================================================

                        トレンドマイクロは3月9日、Wi-Fi利用時の通信を暗号化し保護するスマホ・タブレット向けのセキュリティアプリ「フリーWi-Fiプロテクション」（iOS/Android）の発売を開始すると発表した。1年版ライセンスは2900円（税込）で、2年版ライセンスは5000円（税込）。

                        　フリーWi-Fiプロテクションは、App Storeおよび、Google Playにて販売され、既に提供しているスマホ・タブレット向け総合セキュリティ対策アプリ「ウイルスバスター モバイル」と併用することで、不正アプリや危険なウェブサイトからの保護に加え、通信の盗み見を防ぐことができる。

                        　2020年の東京オリンピック・パラリンピックの開催などを見据え、フリーWi-Fi（公衆無線LAN）の設置が促進され、フリーWi-Fiの利用者も増加している。

                        　一方で、脆弱な設定のフリーWi-Fiや攻撃者が設置した偽のフリーWi-Fiへの接続などによる情報漏えい、通信の盗み見などのセキュリティリスクが危惧されているという。

                        　正規事業者が提供する安全性の高いフリーWi-Fiのほかにも、通信を暗号化していない安全性の低いフリーWi-Fi、さらにはサイバー犯罪者が設置したフリーWi-Fiなどさまざまなものが混在している。また、利用者は、接続する前にひとつひとつ安全性を確認するのは難しい状況だとしている。

                        　トレンドマイクロがスマートフォン保持者でフリーWi-Fiの利用経験がある人に実施した調査では、回答者の約85％が安全なフリーWi-Fiと危険なフリーWi-Fiは「見分けられない」と回答。さらに、約65％がフリーWi-Fiの利用に不安を感じていると回答している。

                        　こうした環境の変化やユーザの状況を鑑み、フリーWi-Fiプロテクションの提供を開始する。同アプリをインストールすることで利用者は、万が一安全性の低いフリーWi-Fiのアクセスポイントに接続してしまった場合でも、その通信を暗号化でき、通信の盗み見やそれによる情報漏えいのリスクを低減できるようになる。

                        　具体的には、フリーWi-Fi利用時に、スマートフォンがフリーWi-Fiプロテクションインフラに接続することにより、フリーWi-Fiのアクセスポイントを介した通信がVPN（Virtual Private Network）で暗号化される。これにより利用者は、第三者から通信を傍受されることやデータの情報漏えいを防ぐことが可能。さらに、かんたん自動接続の機能により、通信を暗号化していない安全性が低いフリーWi-Fi接続時や利用者が指定したWi-Fiへ接続する際に、自動的に通信を暗号化し、利用者の通信を保護する。

                        　また、フリーWi-Fiプロテクションインフラと、莫大なセキュリティ情報のビッグデータを保有するクラウド型セキュリティ技術基盤「Trend Micro Smart Protection Network」（SPN）が連携することで、フリーWi-Fiプロテクションインフラを経由してインターネットを利用する際に、利用者がフィッシング詐欺サイトや偽サイトなどへの不正サイトへアクセスすることをブロックできるという。

                        ================================================ FILE: test/test-pages/yahoo-4/source.html ================================================ トレンドマイクロ、公衆無線LANを安全に使うためのアプリ「フリーWi-Fiプロテクション」 （CNET Japan） - Yahoo!ニュース
                        ここから本文です

                        トレンドマイクロ、公衆無線LANを安全に使うためのアプリ「フリーWi-Fiプロテクション」

                        CNET Japan 3/9(木) 18:45配信

                        トレンドマイクロは3月9日、Wi-Fi利用時の通信を暗号化し保護するスマホ・タブレット向けのセキュリティアプリ「フリーWi-Fiプロテクション」（iOS/Android）の発売を開始すると発表した。1年版ライセンスは2900円（税込）で、2年版ライセンスは5000円（税込）。

                        　フリーWi-Fiプロテクションは、App Storeおよび、Google Playにて販売され、既に提供しているスマホ・タブレット向け総合セキュリティ対策アプリ「ウイルスバスター モバイル」と併用することで、不正アプリや危険なウェブサイトからの保護に加え、通信の盗み見を防ぐことができる。

                        　2020年の東京オリンピック・パラリンピックの開催などを見据え、フリーWi-Fi（公衆無線LAN）の設置が促進され、フリーWi-Fiの利用者も増加している。

                        　一方で、脆弱な設定のフリーWi-Fiや攻撃者が設置した偽のフリーWi-Fiへの接続などによる情報漏えい、通信の盗み見などのセキュリティリスクが危惧されているという。

                        　正規事業者が提供する安全性の高いフリーWi-Fiのほかにも、通信を暗号化していない安全性の低いフリーWi-Fi、さらにはサイバー犯罪者が設置したフリーWi-Fiなどさまざまなものが混在している。また、利用者は、接続する前にひとつひとつ安全性を確認するのは難しい状況だとしている。

                        　トレンドマイクロがスマートフォン保持者でフリーWi-Fiの利用経験がある人に実施した調査では、回答者の約85％が安全なフリーWi-Fiと危険なフリーWi-Fiは「見分けられない」と回答。さらに、約65％がフリーWi-Fiの利用に不安を感じていると回答している。

                        　こうした環境の変化やユーザの状況を鑑み、フリーWi-Fiプロテクションの提供を開始する。同アプリをインストールすることで利用者は、万が一安全性の低いフリーWi-Fiのアクセスポイントに接続してしまった場合でも、その通信を暗号化でき、通信の盗み見やそれによる情報漏えいのリスクを低減できるようになる。

                        　具体的には、フリーWi-Fi利用時に、スマートフォンがフリーWi-Fiプロテクションインフラに接続することにより、フリーWi-Fiのアクセスポイントを介した通信がVPN（Virtual Private Network）で暗号化される。これにより利用者は、第三者から通信を傍受されることやデータの情報漏えいを防ぐことが可能。さらに、かんたん自動接続の機能により、通信を暗号化していない安全性が低いフリーWi-Fi接続時や利用者が指定したWi-Fiへ接続する際に、自動的に通信を暗号化し、利用者の通信を保護する。

                        　また、フリーWi-Fiプロテクションインフラと、莫大なセキュリティ情報のビッグデータを保有するクラウド型セキュリティ技術基盤「Trend Micro Smart Protection Network」（SPN）が連携することで、フリーWi-Fiプロテクションインフラを経由してインターネットを利用する際に、利用者がフィッシング詐欺サイトや偽サイトなどへの不正サイトへアクセスすることをブロックできるという。

                        【関連記事】

                        最終更新:3/9(木) 18:45

                        CNET Japan

                        • ※コメントは個人の見解であり、記事提供社と関係はありません。

                        注目のビジネスニュース

                        アクセスランキング（IT総合）

                        1. 1 「テレビで話題になってしまった茶碗」奈良大が分析　「実物を見ず真贋云々に疑問」 ITmedia NEWS 3/3(金) 11:32
                        2. 2 稚内のスーパーが240個のはずのケーキを2400個誤発注 → Twitterで助けを求める → 2時間で完売 ねとらぼ 3/3(金) 13:23
                        3. 3 就活生のヒール、ぶっちゃけ気になる？　金融・メーカー…企業に聞いたら、同じ答えがずらり withnews 3/8(水) 7:01
                        4. 4 稚内でケーキ2400個誤発注…でも2時間で完売　常連の支えと従業員に感謝　人口3万5千人、最北の市で withnews 3/7(火) 7:00
                        5. 5 客室乗務員による「空の旅の知られざる真実」 BUSINESS INSIDER JAPAN 3/4(土) 20:10

                        雑誌アクセスランキング （IT・科学）

                        1. 1 ＜発覚＞巣立ち後も親のすねをかじるペンギン ナショナル ジオグラフィック日本版 3/9(木) 7:20
                        2. 2 100均で見つけた意外と便利なスマホグッズ大図鑑 ＠ＤＩＭＥ 3/5(日) 8:10
                        3. 3 コロンビアでジカウイルスによる小頭症患者が少ない「恐ろしい理由」 WIRED.jp 3/7(火) 18:20
                        4. 4 パジャマはもういらない。裸で寝るとこんなにもいいことが ライフハッカー［日本版］ 3/8(水) 13:30
                        5. 5 キッズケータイより安価に子どもを見守ることができるMeitrackのSIMフリーGPSトラッカー ＠ＤＩＭＥ 3/9(木) 7:10
                        本文はここまでです このページの先頭へ

                        お得情報

                        その他のキャンペーン

                        ================================================ FILE: test/test-pages/youth/expected-metadata.json ================================================ { "title": "海外留学生看两会：出国前后关注点大不同_教育频道_中国青年网", "byline": "青网校园崔宁宁", "dir": null, "excerpt": "图为马素湘在澳大利亚悉尼游玩时的近影。出国前后关注点大不同出国前：政治科目会出啥考题？出国后：国家未来将如何发展？在采访中，我们了解到不少学子在出国前就每年守在电脑前观看两会直播。但是，随着年龄和阅历的增长，学子对两会的关注点在出国前后发生了很大的变化。在法国里昂国立应用科学院留学的卢宇表示，他还是个中学生时，就开始关注两会了。“我高中毕业后就出国留学了。", "siteName": null, "publishedTime": null, "readerable": true } ================================================ FILE: test/test-pages/youth/expected.html ================================================

                        海外留学生看两会：出国前后关注点大不同

                        图为马素湘在澳大利亚悉尼游玩时的近影。

                        　　出国前后关注点大不同

                        　　出国前：政治科目会出啥考题？

                        　　出国后：国家未来将如何发展？

                        　　 在采访中，我们了解到不少学子在出国前就每年守在电脑前观看两会直播。但是，随着年龄和阅历的增长，学子对两会的关注点在出国前后发生了很大的变化。

                        　　 在法国里昂国立应用科学院留学的卢宇表示，他还是个中学生时，就开始关注两会了。“我高中毕业后就出国留学了。当我还在国内读高中时，对两会的主要关注点落在民生和教育问题上。根据这些内容预测今年政治考点会有哪些变化。”卢宇说，“在国外学习生活了将近10年，我愈发感觉到祖国这些年发生的日新月异的变化；关注点也转移到国家‘一带一路’建设，高端人才引进，中外合作等方面。”

                        　　 无独有偶，英国剑桥大学的李博灏也有着类似的经历。他表示，在国内读本科时，虽然关注过两会，但并不像现在这样，将关注点放在国家社会经济迫切需要解决的难题与问题上。“出国前更关心与我们学生的实际问题以及切身利益相关的议题，比如奖学金、助学金的发放与申请；相关工作行业就业前景等。”

                        　　 在英国求学6年后，李博灏希望能够学有所用，为国家发展过程中遇到的难题寻求解决办法。因此随着课题研究的深入，他更加关注国家和社会目前所面临的挑战，比如中等收入陷阱、供给侧改革、创意创新产业的发展等议题。

                        　　 还有一些学子表示，出国前对两会不太了解，出国后反而对两会热点多了些思考。在澳大利亚墨尔本留学的马素湘说：“想不关注都难啊！刷微博看新闻到处都是两会的消息。而且我现在学的是新闻专业，对世界发生的大小事都会留意。随着年龄、阅历增长，家国情怀也渐长，会关心国家发展的各方面问题。”

                        图为李博灏在瑞士日内日瓦联合国欧洲总部的近影。

                        　　关注点多与所学专业相关

                        　　法学专业热议法定婚龄 很多人关心供给侧改革

                        　　 在谈及对两会的哪些话题比较感兴趣时，卢宇表示：“近几年，国内雾霾现象时有发生。而我本身是学习能源环境专业的，所以每年都对两会上政府工作报告和代表提案中有关环保和新能源政策的部分很感兴趣。今年两会提案中有几份关于大力发展清洁能源汽车的。我认为这对节能减排和防治雾霾都有积极作用，但也要注意加强电池和电动机等关键技术的研发。”

                        　　 对此提案，卢宇有着自己的看法，“百花齐放的局面固然可喜，但也不能一哄而上，国家应该提高行业准入门槛，完善新能源汽车准入管理规则，从源头上进行制度创新，将一些不具备新能源汽车生产资质的厂家淘汰出局，并高度关注电池系统安全问题，严格执行充电桩生产的国际标准。”

                        　　 马素湘表示，“出国读研之前，我在国内学习法学，因此对相关的问题比较感兴趣。今年两会上人大代表黄细花提出把法定婚龄降低到18岁的提案；而在微博的热搜榜上，一本儿童性教育读物引起了极大的争议。我认为降低婚龄并不适合我国国情。因为性教育的缺乏导致我国大部分人在18岁之前没有接受过完整的性教育，思想行动上也不够成熟，如何能够对自己的人生和自己的另一半负责？所以我希望能有人大代表提议在国民儿童阶段完善我国的性教育，而不是为了鼓励生育将法定婚龄提前。”

                        　　 李博灏是英国剑桥大学制造业研究所创新设计管理中心的一名博士。他格外关注的话题是供给侧结构性改革，知识产权保护，消费升级等议题。“我的博士研究课题是关于推动创新设计密集型产业的发展从而帮助中等收入国家克服中等收入陷阱的探索，因此一直十分关注国内关于供给侧改革的相关议题。通过本届两会对于该议题的进一步关注，我希望可以有效地帮助我了解供给侧改革与中等收入陷阱问题目前的发展状况以及解决情况；也希望可以与更多的机构取得联系，并帮助他们了解该议题最前沿的研究与解决方案。”

                        图为卢宇与祖国五星红旗和联合国会旗的合影。

                        　　 两会成为了解国情的窗口

                        　　 盼准确把握国家发展需求 愿寻求机遇回国有所作为

                        　　 不少学子时刻关注着国内动态，寻找回国发展的契机，而两会正是一个提供最新最全面信息的窗口。

                        　　 “对两会的关注使我能够准确把握国家目前迫切需要解决的难题，从而使我的研究具有更多的社会价值。把理论和现实结合起来，将学到的知识、理念与技术有效地融入到国家社会经济发展的实践中去，为国家和社会解决这些问题提供理论与实践依据。”李博灏说。

                        　　 知识产权也是近些年来两会的热门话题。在李博灏看来，知识产权是经济可持续增长和创意创新产业发展的灵魂所在。完善的、与国际接轨的知识产权保护法能够很好地促进国民经济的发展。“如何将欧洲和英国先进的知识产权以及创意产业保护发展经验带回祖国，帮助我们国家推动创意创新驱动的经济发展一直是我在关注与思考的。因此我也会十分关注两会关于国内知识产权的保护，知识产权综合管理改革试点工作，知识产权国际合作，知识产权大数据等的发展现状等问题。”

                        　　 在两会上，全国政协委员张近东提出“当前中国经济的发展正在从数量型向质量型转变，消费升级将成为企业新一轮创新发展的动力。”对此，李博灏认为这也是他关注的问题。他认为：“在消费市场持续扩大的大环境下，如何能够通过促进创新设计产业的发展以及消费品品质的提升，推动国内消费增长并促进其在可持续经济增长中的作用，是一个迫切需要解决的问题。在当前供给侧改革的大环境下，消费升级的重要性越发突显。”

                        　　 作为两会的资深粉，卢宇聊起两会话题充满了期待。“今年是国家‘十三五’规划的关键时期，‘一带一路’建设也在如火如荼地进行中。作为一名中国留学生，我一直都关注着能在哪些领域为国家、为中外合作共赢做出贡献。‘大众创业、万众创新’提出有几年了。全国各省市在吸引留学人才归国创业就业方面纷纷提出了各种优惠政策，但目前大都集中在沿海发达省份，而且主要惠及理工科博士，政策覆盖面还不够广。期待从国家层面设立工作组加强留学人才的统筹协调，完善顶层设计。人文社科类留学人才是未来国家智库的重要后备力量，也应该适当加强对他们的政策鼓励，更好地服务于‘一带一路’国家战略。”

                        　　 卢宇还认为两会应该增设学子代表，列席旁听两会，拓展留学生参政议政渠道。“我相信优秀留学生的国际化视野必将为家乡建设带来新的思路，增添新的活力。”卢宇恳切地说。

                        ================================================ FILE: test/test-pages/youth/source.html ================================================ 海外留学生看两会：出国前后关注点大不同_教育频道_中国青年网
                        首页|新闻|图片|评论|共青团|娱乐|时尚|财经|军事|体育|创业就业|高校|旅游|发现|视频|游戏|汽车|青春励志
                        海外留学生看两会：出国前后关注点大不同

                        发稿时间：2017-03-10 08:42:00 来源： 人民日报海外版 中国青年网

                        海外留学生看两会：出国前后关注点大不同

                        图为马素湘在澳大利亚悉尼游玩时的近影。

                        　　出国前后关注点大不同

                        　　出国前：政治科目会出啥考题？

                        　　出国后：国家未来将如何发展？

                        　　 在采访中，我们了解到不少学子在出国前就每年守在电脑前观看两会直播。但是，随着年龄和阅历的增长，学子对两会的关注点在出国前后发生了很大的变化。

                        　　 在法国里昂国立应用科学院留学的卢宇表示，他还是个中学生时，就开始关注两会了。“我高中毕业后就出国留学了。当我还在国内读高中时，对两会的主要关注点落在民生和教育问题上。根据这些内容预测今年政治考点会有哪些变化。”卢宇说，“在国外学习生活了将近10年，我愈发感觉到祖国这些年发生的日新月异的变化；关注点也转移到国家‘一带一路’建设，高端人才引进，中外合作等方面。”

                        　　 无独有偶，英国剑桥大学的李博灏也有着类似的经历。他表示，在国内读本科时，虽然关注过两会，但并不像现在这样，将关注点放在国家社会经济迫切需要解决的难题与问题上。“出国前更关心与我们学生的实际问题以及切身利益相关的议题，比如奖学金、助学金的发放与申请；相关工作行业就业前景等。”

                        　　 在英国求学6年后，李博灏希望能够学有所用，为国家发展过程中遇到的难题寻求解决办法。因此随着课题研究的深入，他更加关注国家和社会目前所面临的挑战，比如中等收入陷阱、供给侧改革、创意创新产业的发展等议题。

                        　　 还有一些学子表示，出国前对两会不太了解，出国后反而对两会热点多了些思考。在澳大利亚墨尔本留学的马素湘说：“想不关注都难啊！刷微博看新闻到处都是两会的消息。而且我现在学的是新闻专业，对世界发生的大小事都会留意。随着年龄、阅历增长，家国情怀也渐长，会关心国家发展的各方面问题。”

                        图为李博灏在瑞士日内日瓦联合国欧洲总部的近影。

                        　　关注点多与所学专业相关

                        　　法学专业热议法定婚龄 很多人关心供给侧改革

                        　　 在谈及对两会的哪些话题比较感兴趣时，卢宇表示：“近几年，国内雾霾现象时有发生。而我本身是学习能源环境专业的，所以每年都对两会上政府工作报告和代表提案中有关环保和新能源政策的部分很感兴趣。今年两会提案中有几份关于大力发展清洁能源汽车的。我认为这对节能减排和防治雾霾都有积极作用，但也要注意加强电池和电动机等关键技术的研发。”

                        　　 对此提案，卢宇有着自己的看法，“百花齐放的局面固然可喜，但也不能一哄而上，国家应该提高行业准入门槛，完善新能源汽车准入管理规则，从源头上进行制度创新，将一些不具备新能源汽车生产资质的厂家淘汰出局，并高度关注电池系统安全问题，严格执行充电桩生产的国际标准。”

                        　　 马素湘表示，“出国读研之前，我在国内学习法学，因此对相关的问题比较感兴趣。今年两会上人大代表黄细花提出把法定婚龄降低到18岁的提案；而在微博的热搜榜上，一本儿童性教育读物引起了极大的争议。我认为降低婚龄并不适合我国国情。因为性教育的缺乏导致我国大部分人在18岁之前没有接受过完整的性教育，思想行动上也不够成熟，如何能够对自己的人生和自己的另一半负责？所以我希望能有人大代表提议在国民儿童阶段完善我国的性教育，而不是为了鼓励生育将法定婚龄提前。”

                        　　 李博灏是英国剑桥大学制造业研究所创新设计管理中心的一名博士。他格外关注的话题是供给侧结构性改革，知识产权保护，消费升级等议题。“我的博士研究课题是关于推动创新设计密集型产业的发展从而帮助中等收入国家克服中等收入陷阱的探索，因此一直十分关注国内关于供给侧改革的相关议题。通过本届两会对于该议题的进一步关注，我希望可以有效地帮助我了解供给侧改革与中等收入陷阱问题目前的发展状况以及解决情况；也希望可以与更多的机构取得联系，并帮助他们了解该议题最前沿的研究与解决方案。”

                        图为卢宇与祖国五星红旗和联合国会旗的合影。

                        　　 两会成为了解国情的窗口

                        　　 盼准确把握国家发展需求 愿寻求机遇回国有所作为

                        　　 不少学子时刻关注着国内动态，寻找回国发展的契机，而两会正是一个提供最新最全面信息的窗口。

                        　　 “对两会的关注使我能够准确把握国家目前迫切需要解决的难题，从而使我的研究具有更多的社会价值。把理论和现实结合起来，将学到的知识、理念与技术有效地融入到国家社会经济发展的实践中去，为国家和社会解决这些问题提供理论与实践依据。”李博灏说。

                        　　 知识产权也是近些年来两会的热门话题。在李博灏看来，知识产权是经济可持续增长和创意创新产业发展的灵魂所在。完善的、与国际接轨的知识产权保护法能够很好地促进国民经济的发展。“如何将欧洲和英国先进的知识产权以及创意产业保护发展经验带回祖国，帮助我们国家推动创意创新驱动的经济发展一直是我在关注与思考的。因此我也会十分关注两会关于国内知识产权的保护，知识产权综合管理改革试点工作，知识产权国际合作，知识产权大数据等的发展现状等问题。”

                        　　 在两会上，全国政协委员张近东提出“当前中国经济的发展正在从数量型向质量型转变，消费升级将成为企业新一轮创新发展的动力。”对此，李博灏认为这也是他关注的问题。他认为：“在消费市场持续扩大的大环境下，如何能够通过促进创新设计产业的发展以及消费品品质的提升，推动国内消费增长并促进其在可持续经济增长中的作用，是一个迫切需要解决的问题。在当前供给侧改革的大环境下，消费升级的重要性越发突显。”

                        　　 作为两会的资深粉，卢宇聊起两会话题充满了期待。“今年是国家‘十三五’规划的关键时期，‘一带一路’建设也在如火如荼地进行中。作为一名中国留学生，我一直都关注着能在哪些领域为国家、为中外合作共赢做出贡献。‘大众创业、万众创新’提出有几年了。全国各省市在吸引留学人才归国创业就业方面纷纷提出了各种优惠政策，但目前大都集中在沿海发达省份，而且主要惠及理工科博士，政策覆盖面还不够广。期待从国家层面设立工作组加强留学人才的统筹协调，完善顶层设计。人文社科类留学人才是未来国家智库的重要后备力量，也应该适当加强对他们的政策鼓励，更好地服务于‘一带一路’国家战略。”

                        　　 卢宇还认为两会应该增设学子代表，列席旁听两会，拓展留学生参政议政渠道。“我相信优秀留学生的国际化视野必将为家乡建设带来新的思路，增添新的活力。”卢宇恳切地说。

                        责任编辑：崔宁宁
                        返回首页>>
                        收藏文章
                        我的社区
                        表情删除后不可恢复，是否删除
                        取消
                        确定
                        图片正在上传，请稍后...
                        取消上传
                        评论内容为空！
                        还没有评论，快来抢沙发吧！
                        该评论已关闭!
                        畅言
                        去社区看看吧
                        去热评看看吧
                        热图

                        排行

                        热点

                        推荐

                        热搜

                        排行

                        视觉

                        焦点

                        精华

                        推荐

                        女性养生：女人最易老的身体禁区 男人千万别看
                        女性养生：女人最易老的身体...
                        会是神器吗？口香糖大小设备可运行iOS游戏
                        会是神器吗？口香糖大小设备...
                        盘点NBA5大选秀日操作影响 谁是最大赢家
                        盘点NBA5大选秀日操作影响 ...
                        “英超女神”墨西哥戏水 双峰全裸尺度惊人
                        “英超女神”墨西哥戏水 双...
                        杨幂石天琦刘诗诗柳岩 六月银幕御姐齐发力
                        杨幂石天琦刘诗诗柳岩 六月...

                        关于我们｜联系我们｜广告服务｜人才招聘｜中国互联网举报中心　Youth.cn. 请发送qnb至10658000　订阅手机青年报

                        可信网站
                        共青团中央主办　共青团中央网络影视中心承办　版权所有：中国青年网
                        信息网络传播视听节目许可证0105108号　京｜ICP备11020872号－17　京公网安备110105007246
                        x
                        ================================================ FILE: test/test-readability.js ================================================ /* eslint-env node, mocha */ var JSDOM = require("jsdom").JSDOM; var xmlNameValidator = require("xml-name-validator").name; var chai = require("chai"); var sinon = require("sinon"); chai.config.includeStack = true; var expect = chai.expect; var Readability = require("../index").Readability; var JSDOMParser = require("../JSDOMParser"); var prettyPrint = require("./utils").prettyPrint; var testPages = require("./utils").getTestPages(); function reformatError(err) { var formattedError = new Error(err.message); formattedError.stack = err.stack; return formattedError; } function inOrderTraverse(fromNode) { if (fromNode.firstChild) { return fromNode.firstChild; } while (fromNode && !fromNode.nextSibling) { fromNode = fromNode.parentNode; } return fromNode ? fromNode.nextSibling : null; } function inOrderIgnoreEmptyTextNodes(fromNode) { do { fromNode = inOrderTraverse(fromNode); } while (fromNode && fromNode.nodeType == 3 && !fromNode.textContent.trim()); return fromNode; } function traverseDOM(callback, expectedDOM, actualDOM) { var actualNode = actualDOM.documentElement || actualDOM.childNodes[0]; var expectedNode = expectedDOM.documentElement || expectedDOM.childNodes[0]; while (actualNode || expectedNode) { // We'll stop if we don't have both actualNode and expectedNode if (!callback(actualNode, expectedNode)) { break; } actualNode = inOrderIgnoreEmptyTextNodes(actualNode); expectedNode = inOrderIgnoreEmptyTextNodes(expectedNode); } } // Collapse subsequent whitespace like HTML: function htmlTransform(str) { return str.replace(/\s+/g, " "); } function runTestsWithItems( label, domGenerationFn, source, expectedContent, expectedMetadata ) { describe(label, function () { this.timeout(30000); var result; before(function () { try { var doc = domGenerationFn(source); // Provide one class name to preserve, which we know appears in a few // of the test documents. var myReader = new Readability(doc, { classesToPreserve: ["caption"] }); result = myReader.parse(); } catch (err) { throw reformatError(err); } }); it("should return a result object", function () { expect(result).to.include.keys("content", "title", "excerpt", "byline"); }); it("should extract expected content", function () { function nodeStr(n) { if (!n) { return "(no node)"; } if (n.nodeType == 3) { return "#text(" + htmlTransform(n.textContent) + ")"; } if (n.nodeType != 1) { return "some other node type: " + n.nodeType + " with data " + n.data; } var rv = n.localName; if (n.id) { rv += "#" + n.id; } if (n.className) { rv += ".(" + n.className + ")"; } return rv; } function genPath(node) { if (node.id) { return "#" + node.id; } if (node.tagName == "BODY") { return "body"; } var parent = node.parentNode; var parentPath = genPath(parent); var index = Array.prototype.indexOf.call(parent.childNodes, node) + 1; return parentPath + " > " + nodeStr(node) + ":nth-child(" + index + ")"; } function findableNodeDesc(node) { return genPath(node) + "(in: ``" + node.parentNode.innerHTML + "``)"; } function attributesForNode(node) { return Array.from(node.attributes) .filter(function (attr) { return xmlNameValidator(attr.name); }) .map(function (attr) { return attr.name + "=" + attr.value; }); } var actualDOM = domGenerationFn(prettyPrint(result.content)); var expectedDOM = domGenerationFn(prettyPrint(expectedContent)); traverseDOM( function (actualNode, expectedNode) { if (actualNode && expectedNode) { var actualDesc = nodeStr(actualNode); var expectedDesc = nodeStr(expectedNode); if (actualDesc != expectedDesc) { expect(actualDesc, findableNodeDesc(actualNode)).eql( expectedDesc ); return false; } // Compare text for text nodes: if (actualNode.nodeType == 3) { var actualText = htmlTransform(actualNode.textContent); var expectedText = htmlTransform(expectedNode.textContent); expect(actualText, findableNodeDesc(actualNode)).eql( expectedText ); if (actualText != expectedText) { return false; } // Compare attributes for element nodes: } else if (actualNode.nodeType == 1) { var actualNodeAttributes = attributesForNode(actualNode); var expectedNodeAttributes = attributesForNode(expectedNode); var desc = "node " + nodeStr(actualNode) + " attributes (" + actualNodeAttributes.join(",") + ") should match (" + expectedNodeAttributes.join(",") + ") 1"; expect(actualNodeAttributes.length, desc).eql( expectedNodeAttributes.length ); for (var i = 0; i < actualNodeAttributes.length; i++) { var attr = actualNodeAttributes[i].name; var actualValue = actualNode.getAttribute(attr); var expectedValue = expectedNode.getAttribute(attr); expect( expectedValue, "node (" + findableNodeDesc(actualNode) + ") attribute " + attr + " should match" ).eql(actualValue); } } } else { expect( nodeStr(actualNode), "Should have a node from both DOMs" ).eql(nodeStr(expectedNode)); return false; } return true; }, actualDOM, expectedDOM ); }); it("should extract expected title", function () { expect(result.title).eql(expectedMetadata.title); }); it("should extract expected byline", function () { expect(result.byline).eql(expectedMetadata.byline); }); it("should extract expected excerpt", function () { expect(result.excerpt).eql(expectedMetadata.excerpt); }); it("should extract expected site name", function () { expect(result.siteName).eql(expectedMetadata.siteName); }); expectedMetadata.dir && it("should extract expected direction", function () { expect(result.dir).eql(expectedMetadata.dir); }); expectedMetadata.lang && it("should extract expected language", function () { expect(result.lang).eql(expectedMetadata.lang); }); expectedMetadata.publishedTime && it("should extract expected published time", function () { expect(result.publishedTime).eql(expectedMetadata.publishedTime); }); }); } function removeCommentNodesRecursively(node) { for (var i = node.childNodes.length - 1; i >= 0; i--) { var child = node.childNodes[i]; if (child.nodeType === child.COMMENT_NODE) { node.removeChild(child); } else if (child.nodeType === child.ELEMENT_NODE) { removeCommentNodesRecursively(child); } } } describe("Readability API", function () { describe("#constructor", function () { var doc = new JSDOMParser().parse("
                        yo
                        "); it("should accept a debug option", function () { expect(new Readability(doc)._debug).eql(false); expect(new Readability(doc, { debug: true })._debug).eql(true); }); it("should accept a nbTopCandidates option", function () { expect(new Readability(doc)._nbTopCandidates).eql(5); expect( new Readability(doc, { nbTopCandidates: 42 })._nbTopCandidates ).eql(42); }); it("should accept a maxElemsToParse option", function () { expect(new Readability(doc)._maxElemsToParse).eql(0); expect( new Readability(doc, { maxElemsToParse: 42 })._maxElemsToParse ).eql(42); }); it("should accept a keepClasses option", function () { expect(new Readability(doc)._keepClasses).eql(false); expect(new Readability(doc, { keepClasses: true })._keepClasses).eql( true ); expect(new Readability(doc, { keepClasses: false })._keepClasses).eql( false ); }); it("should accept a allowedVideoRegex option or default it", function () { expect(new Readability(doc)._allowedVideoRegex).eql( Readability.prototype.REGEXPS.videos ); const allowedVideoRegex = /\/\/mydomain.com\/.*'/; expect( new Readability(doc, { allowedVideoRegex })._allowedVideoRegex ).eql(allowedVideoRegex); }); }); describe("#parse", function () { var exampleSource = testPages[0].source; it("shouldn't parse oversized documents as per configuration", function () { var doc = new JSDOMParser().parse("
                        yo
                        "); expect(function () { new Readability(doc, { maxElemsToParse: 1 }).parse(); }).to.Throw("Aborting parsing document; 2 elements found"); }); it("should run _cleanClasses with default configuration", function () { var doc = new JSDOMParser().parse(exampleSource); var parser = new Readability(doc); parser._cleanClasses = sinon.fake(); parser.parse(); expect(parser._cleanClasses.called).eql(true); }); it("should run _cleanClasses when option keepClasses = false", function () { var doc = new JSDOMParser().parse(exampleSource); var parser = new Readability(doc, { keepClasses: false }); parser._cleanClasses = sinon.fake(); parser.parse(); expect(parser._cleanClasses.called).eql(true); }); it("shouldn't run _cleanClasses when option keepClasses = true", function () { var doc = new JSDOMParser().parse(exampleSource); var parser = new Readability(doc, { keepClasses: true }); parser._cleanClasses = sinon.fake(); parser.parse(); expect(parser._cleanClasses.called).eql(false); }); it("should use custom content serializer sent as option", function () { var dom = new JSDOM("My cat: "); var expected_xhtml = '
                        My cat:
                        '; var xml = new dom.window.XMLSerializer(); var content = new Readability(dom.window.document, { serializer(el) { return xml.serializeToString(el.firstChild); }, }).parse().content; expect(content).eql(expected_xhtml); }); it("should use custom video regex sent as option", function () { var dom = new JSDOM( "

                        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mollis leo lacus, vitae semper nisl ullamcorper ut.

                        " + '' ); var expected_xhtml = '
                        ' + "

                        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mollis leo lacus, vitae semper nisl ullamcorper ut.

                        " + '' + "
                        "; var content = new Readability(dom.window.document, { charThreshold: 20, allowedVideoRegex: /.*mycustomdomain.com.*/, }).parse().content; expect(content).eql(expected_xhtml); }); }); }); describe("Test pages", function () { testPages.forEach(function (testPage) { describe(testPage.dir, function () { var uri = "http://fakehost/test/page.html"; runTestsWithItems( "jsdom", function (source) { var doc = new JSDOM(source, { url: uri, }).window.document; removeCommentNodesRecursively(doc); return doc; }, testPage.source, testPage.expectedContent, testPage.expectedMetadata ); runTestsWithItems( "JSDOMParser", function (source) { var parser = new JSDOMParser(); var doc = parser.parse(source, uri); if (parser.errorState) { console.error("Parsing this DOM caused errors:", parser.errorState); return null; } return doc; }, testPage.source, testPage.expectedContent, testPage.expectedMetadata ); }); }); }); ================================================ FILE: test/utils.js ================================================ /* eslint-env node */ var path = require("path"); var fs = require("fs"); var prettyPrint = require("js-beautify").html; function readFile(filePath) { return fs.readFileSync(filePath, { encoding: "utf-8" }).trim(); } function readJSON(jsonPath) { return JSON.parse(readFile(jsonPath)); } var testPageRoot = path.join(__dirname, "test-pages"); exports.getTestPages = function () { return fs.readdirSync(testPageRoot).map(function (dir) { return { dir, source: readFile(path.join(testPageRoot, dir, "source.html")), expectedContent: readFile(path.join(testPageRoot, dir, "expected.html")), expectedMetadata: readJSON( path.join(testPageRoot, dir, "expected-metadata.json") ), }; }); }; exports.prettyPrint = function (html) { return prettyPrint(html, { indent_size: 4, indent_char: " ", indent_level: 0, indent_with_tabs: false, preserve_newlines: false, break_chained_methods: false, eval_code: false, unescape_strings: false, wrap_line_length: 0, wrap_attributes: "auto", wrap_attributes_indent_size: 4, }); };