Each Node is still the same DOM connected Node.
\n\n @return a new ArrayList containing the nodes in this list\n @see #Nodes(List)\n */\n public ArrayList asList() {\n return new ArrayList<>(this);\n }\n\n /**\n Remove each matched node from the DOM.\n The nodes will still be retained in this list, in case further processing of them is desired.
\n \n E.g. HTML: {@code
Hello
there
![]()
\n doc.select(\"p\").remove();
\n HTML = {@code ![]()
}\n \n Note that this method should not be used to clean user-submitted HTML; rather, use {@link org.jsoup.safety.Cleaner}\n to clean HTML.\n\n @return this, for chaining\n @see Element#empty()\n @see Elements#empty()\n @see #clear()\n */\n public Nodes remove() {\n for (T node : this) {\n node.remove();\n }\n return this;\n }\n\n /**\n Get the combined outer HTML of all matched nodes.\n\n @return string of all node's outer HTML.\n @see Elements#text()\n @see Elements#html()\n */\n public String outerHtml() {\n return stream()\n .map(Node::outerHtml)\n .collect(StringUtil.joining(\"\\n\"));\n }\n\n /**\n Get the combined outer HTML of all matched nodes. Alias of {@link #outerHtml()}.\n\n @return string of all the node's outer HTML.\n @see Elements#text()\n @see #outerHtml()\n */\n @Override\n public String toString() {\n return outerHtml();\n }\n\n /**\n Insert the supplied HTML before each matched node's outer HTML.\n\n @param html HTML to insert before each node\n @return this, for chaining\n @see Element#before(String)\n */\n public Nodes before(String html) {\n for (T node : this) {\n node.before(html);\n }\n return this;\n }\n\n /**\n Insert the supplied HTML after each matched nodes's outer HTML.\n\n @param html HTML to insert after each node\n @return this, for chaining\n @see Element#after(String)\n */\n public Nodes after(String html) {\n for (T node : this) {\n node.after(html);\n }\n return this;\n }\n\n /**\n Wrap the supplied HTML around each matched node. For example, with HTML\n {@code This is Jsoup
},\n doc.select(\"b\").wrap(\"<i></i>\");\n becomes {@code This is jsoup
}\n @param html HTML to wrap around each node, e.g. {@code }. Can be arbitrarily deep.\n @return this (for chaining)\n @see Element#wrap\n */\n public Nodes wrap(String html) {\n Validate.notEmpty(html);\n for (T node : this) {\n node.wrap(html);\n }\n return this;\n }\n\n // list-like methods\n /**\n Get the first matched element.\n @return The first matched element, or null if contents is empty.\n */\n public @Nullable T first() {\n return isEmpty() ? null : get(0);\n }\n\n /**\n Get the last matched element.\n @return The last matched element, or null if contents is empty.\n */\n public @Nullable T last() {\n return isEmpty() ? null : get(size() - 1);\n }\n\n // ArrayList methods that update the DOM:\n\n /**\n Replace the node at the specified index in this list, and in the DOM.\n\n @param index index of the node to replace\n @param node node to be stored at the specified position\n @return the old Node at this index\n */\n @Override\n public T set(int index, T node) {\n Validate.notNull(node);\n T old = super.set(index, node);\n old.replaceWith(node);\n return old;\n }\n\n /**\n Remove the node at the specified index in this list, and from the DOM.\n\n @param index the index of the node to be removed\n @return the old node at this index\n @see #deselect(int)\n */\n @Override\n public T remove(int index) {\n T old = super.remove(index);\n old.remove();\n return old;\n }\n\n /**\n Remove the specified node from this list, and from the DOM.\n\n @param o node to be removed from this list, if present\n @return if this list contained the Node\n @see #deselect(Object)\n */\n @Override\n public boolean remove(Object o) {\n int index = super.indexOf(o);\n if (index == -1) {\n return false;\n } else {\n remove(index);\n return true;\n }\n }\n\n /**\n Remove the node at the specified index in this list, but not from the DOM.\n\n @param index the index of the node to be removed\n @return the old node at this index\n @see #remove(int)\n */\n public T deselect(int index) {\n return super.remove(index);\n }\n\n /**\n Remove the specified node from this list, but not from the DOM.\n\n @param o node to be removed from this list, if present\n @return if this list contained the Node\n @see #remove(Object)\n */\n public boolean deselect(Object o) {\n return super.remove(o);\n }\n\n /**\n Removes all the nodes from this list, and each of them from the DOM.\n\n @see #deselectAll()\n */\n @Override\n public void clear() {\n remove();\n super.clear();\n }\n\n /**\n Like {@link #clear()}, removes all the nodes from this list, but not from the DOM.\n\n @see #clear()\n */\n public void deselectAll() {\n super.clear();\n }\n\n /**\n Removes from this list, and from the DOM, each of the nodes that are contained in the specified collection and are\n in this list.\n\n @param c collection containing nodes to be removed from this list\n @return {@code true} if nodes were removed from this list\n */\n @Override\n public boolean removeAll(Collection c) {\n boolean anyRemoved = false;\n for (Object o : c) {\n anyRemoved |= this.remove(o);\n }\n return anyRemoved;\n }\n\n /**\n Retain in this list, and in the DOM, only the nodes that are in the specified collection and are in this list. In\n other words, remove nodes from this list and the DOM any item that is in this list but not in the specified\n collection.\n\n @param toRemove collection containing nodes to be retained in this list\n @return {@code true} if nodes were removed from this list\n @since 1.17.1\n */\n @Override\n public boolean retainAll(Collection toRemove) {\n boolean anyRemoved = false;\n for (Iterator it = this.iterator(); it.hasNext(); ) {\n T el = it.next();\n if (!toRemove.contains(el)) {\n it.remove();\n anyRemoved = true;\n }\n }\n return anyRemoved;\n }\n\n /**\n Remove from the list, and from the DOM, all nodes in this list that mach the given predicate.\n\n @param filter a predicate which returns {@code true} for nodes to be removed\n @return {@code true} if nodes were removed from this list\n */\n @Override\n public boolean removeIf(Predicate filter) {\n boolean anyRemoved = false;\n for (Iterator it = this.iterator(); it.hasNext(); ) {\n T node = it.next();\n if (filter.test(node)) {\n it.remove();\n anyRemoved = true;\n }\n }\n return anyRemoved;\n }\n\n /**\n Replace each node in this list with the result of the operator, and update the DOM.\n\n @param operator the operator to apply to each node\n */\n @Override\n public void replaceAll(UnaryOperator operator) {\n for (int i = 0; i < this.size(); i++) {\n this.set(i, operator.apply(this.get(i)));\n }\n }\n}\n"
},
{
"path": "src/main/java/org/jsoup/select/QueryParser.java",
"content": "package org.jsoup.select;\n\nimport org.jsoup.helper.Regex;\nimport org.jsoup.internal.StringUtil;\nimport org.jsoup.helper.Validate;\nimport org.jsoup.nodes.CDataNode;\nimport org.jsoup.nodes.Comment;\nimport org.jsoup.nodes.DataNode;\nimport org.jsoup.nodes.LeafNode;\nimport org.jsoup.nodes.Node;\nimport org.jsoup.nodes.TextNode;\nimport org.jsoup.parser.TokenQueue;\nimport org.jspecify.annotations.Nullable;\n\nimport java.util.function.Function;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport static org.jsoup.select.StructuralEvaluator.ImmediateParentRun;\nimport static org.jsoup.internal.Normalizer.normalize;\n\n/**\n * Parses a CSS selector into an Evaluator tree.\n */\npublic class QueryParser implements AutoCloseable {\n private final static char[] Combinators = {'>', '+', '~'}; // ' ' is also a combinator, but found implicitly\n private final static String[] AttributeEvals = new String[]{\"=\", \"!=\", \"^=\", \"$=\", \"*=\", \"~=\"};\n private final static char[] SequenceEnders = {',', ')'};\n\n private final TokenQueue tq;\n private final String query;\n private boolean inNodeContext; // ::comment:contains should act on node value, vs element text\n\n /**\n * Create a new QueryParser.\n * @param query CSS query\n */\n private QueryParser(String query) {\n Validate.notEmpty(query);\n query = query.trim();\n this.query = query;\n this.tq = new TokenQueue(query);\n }\n\n /**\n Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to\n parse it once and reuse the Evaluator.\n\n @param query CSS query\n @return Evaluator\n @see Selector selector query syntax\n @throws Selector.SelectorParseException if the CSS query is invalid\n */\n public static Evaluator parse(String query) {\n try (QueryParser p = new QueryParser(query)) {\n return p.parse();\n } catch (IllegalArgumentException e) {\n throw new Selector.SelectorParseException(e.getMessage());\n }\n }\n\n /**\n Parse the query. We use this simplified expression of the grammar:\n \n SelectorGroup ::= Selector (',' Selector)*\n Selector ::= [ Combinator ] SimpleSequence ( Combinator SimpleSequence )*\n SimpleSequence ::= [ TypeSelector ] ( ID | Class | Attribute | Pseudo )*\n Pseudo ::= ':' Name [ '(' SelectorGroup ')' ]\n Combinator ::= S+ // descendant (whitespace)\n | '>' // child\n | '+' // adjacent sibling\n | '~' // general sibling\n \n\n See selectors-4 for the real thing\n */\n Evaluator parse() {\n Evaluator eval = parseSelectorGroup();\n tq.consumeWhitespace();\n if (!tq.isEmpty())\n throw new Selector.SelectorParseException(\"Could not parse query '%s': unexpected token at '%s'\", query, tq.remainder());\n return eval;\n }\n\n Evaluator parseSelectorGroup() {\n // SelectorGroup. Into an Or if > 1 Selector\n Evaluator left = parseSelector();\n while (tq.matchChomp(',')) {\n Evaluator right = parseSelector();\n left = or(left, right);\n }\n return left;\n }\n\n Evaluator parseSelector() {\n // Selector ::= [ Combinator ] SimpleSequence ( Combinator SimpleSequence )*\n tq.consumeWhitespace();\n\n Evaluator left;\n if (tq.matchesAny(Combinators)) {\n // e.g. query is \"> div\"; left side is root element\n left = new StructuralEvaluator.Root();\n } else {\n left = parseSimpleSequence();\n }\n\n while (true) {\n char combinator = 0;\n if (tq.consumeWhitespace())\n combinator = ' '; // maybe descendant?\n if (tq.matchesAny(Combinators)) // no, explicit\n combinator = tq.consume();\n else if (tq.matchesAny(SequenceEnders)) // , - space after simple like \"foo , bar\"; ) - close of :has()\n break;\n\n if (combinator != 0) {\n Evaluator right = parseSimpleSequence();\n left = combinator(left, combinator, right);\n } else {\n break;\n }\n }\n return left;\n }\n\n Evaluator parseSimpleSequence() {\n // SimpleSequence ::= TypeSelector? ( Hash | Class | Pseudo )*\n Evaluator left = null;\n tq.consumeWhitespace();\n\n // one optional type selector\n if (tq.matchesWord() || tq.matches(\"*|\"))\n left = byTag();\n else if (tq.matchChomp('*'))\n left = new Evaluator.AllElements();\n\n // zero or more subclasses (#, ., [)\n while(true) {\n Evaluator right = parseSubclass();\n if (right != null) {\n left = and(left, right);\n }\n else break; // no more simple tokens\n }\n\n if (left == null)\n throw new Selector.SelectorParseException(\"Could not parse query '%s': unexpected token at '%s'\", query, tq.remainder());\n return left;\n }\n\n static Evaluator combinator(Evaluator left, char combinator, Evaluator right) {\n switch (combinator) {\n case '>':\n ImmediateParentRun run = left instanceof ImmediateParentRun ?\n (ImmediateParentRun) left : new ImmediateParentRun(left);\n run.add(right);\n return run;\n case ' ':\n return and(new StructuralEvaluator.Ancestor(left), right);\n case '+':\n return and(new StructuralEvaluator.ImmediatePreviousSibling(left), right);\n case '~':\n return and(new StructuralEvaluator.PreviousSibling(left), right);\n default:\n throw new Selector.SelectorParseException(\"Unknown combinator '%s'\", combinator);\n }\n }\n\n @Nullable Evaluator parseSubclass() {\n // Subclass: ID | Class | Attribute | Pseudo\n if (tq.matchChomp('#')) return byId();\n else if (tq.matchChomp('.')) return byClass();\n else if (tq.matches('[')) return byAttribute();\n else if (tq.matchChomp(\"::\")) return parseNodeSelector(); // ::comment etc\n else if (tq.matchChomp(':')) return parsePseudoSelector();\n else return null;\n }\n\n /** Merge two evals into an Or. */\n static Evaluator or(Evaluator left, Evaluator right) {\n if (left instanceof CombiningEvaluator.Or) {\n ((CombiningEvaluator.Or) left).add(right);\n return left;\n }\n return new CombiningEvaluator.Or(left, right);\n }\n\n /** Merge two evals into an And. */\n static Evaluator and(@Nullable Evaluator left, Evaluator right) {\n if (left == null) return right;\n if (left instanceof CombiningEvaluator.And) {\n ((CombiningEvaluator.And) left).add(right);\n return left;\n }\n return new CombiningEvaluator.And(left, right);\n }\n\n private Evaluator parsePseudoSelector() {\n final String pseudo = tq.consumeCssIdentifier();\n switch (pseudo) {\n case \"lt\":\n return new Evaluator.IndexLessThan(consumeIndex());\n case \"gt\":\n return new Evaluator.IndexGreaterThan(consumeIndex());\n case \"eq\":\n return new Evaluator.IndexEquals(consumeIndex());\n case \"has\":\n return has();\n case \"is\":\n return is();\n case \"contains\":\n return contains(false);\n case \"containsOwn\":\n return contains(true);\n case \"containsWholeText\":\n return containsWholeText(false);\n case \"containsWholeOwnText\":\n return containsWholeText(true);\n case \"containsData\":\n return containsData();\n case \"matches\":\n return matches(false);\n case \"matchesOwn\":\n return matches(true);\n case \"matchesWholeText\":\n return matchesWholeText(false);\n case \"matchesWholeOwnText\":\n return matchesWholeText(true);\n case \"not\":\n return not();\n case \"nth-child\":\n return cssNthChild(false, false);\n case \"nth-last-child\":\n return cssNthChild(true, false);\n case \"nth-of-type\":\n return cssNthChild(false, true);\n case \"nth-last-of-type\":\n return cssNthChild(true, true);\n case \"first-child\":\n return new Evaluator.IsFirstChild();\n case \"last-child\":\n return new Evaluator.IsLastChild();\n case \"first-of-type\":\n return new Evaluator.IsFirstOfType();\n case \"last-of-type\":\n return new Evaluator.IsLastOfType();\n case \"only-child\":\n return new Evaluator.IsOnlyChild();\n case \"only-of-type\":\n return new Evaluator.IsOnlyOfType();\n case \"empty\":\n return new Evaluator.IsEmpty();\n case \"blank\":\n return new NodeEvaluator.BlankValue();\n case \"root\":\n return new Evaluator.IsRoot();\n case \"matchText\":\n return new Evaluator.MatchText();\n default:\n throw new Selector.SelectorParseException(\"Could not parse query '%s': unexpected token at '%s'\", query, tq.remainder());\n }\n }\n\n // ::comment etc\n private Evaluator parseNodeSelector() {\n final String pseudo = tq.consumeCssIdentifier();\n inNodeContext = true; // Enter node context\n\n Evaluator left;\n switch (pseudo) {\n case \"node\":\n left = new NodeEvaluator.InstanceType(Node.class, pseudo);\n break;\n case \"leafnode\":\n left = new NodeEvaluator.InstanceType(LeafNode.class, pseudo);\n break;\n case \"text\":\n left = new NodeEvaluator.InstanceType(TextNode.class, pseudo);\n break;\n case \"comment\":\n left = new NodeEvaluator.InstanceType(Comment.class, pseudo);\n break;\n case \"data\":\n left = new NodeEvaluator.InstanceType(DataNode.class, pseudo);\n break;\n case \"cdata\":\n left = new NodeEvaluator.InstanceType(CDataNode.class, pseudo);\n break;\n default:\n throw new Selector.SelectorParseException(\n \"Could not parse query '%s': unknown node type '::%s'\", query, pseudo);\n }\n\n // Handle following subclasses in node context (like ::comment:contains())\n Evaluator right;\n while ((right = parseSubclass()) != null) {\n left = and(left, right);\n }\n\n inNodeContext = false;\n return left;\n }\n\n private Evaluator byId() {\n String id = tq.consumeCssIdentifier();\n Validate.notEmpty(id);\n return new Evaluator.Id(id);\n }\n\n private Evaluator byClass() {\n String className = tq.consumeCssIdentifier();\n Validate.notEmpty(className);\n return new Evaluator.Class(className.trim());\n }\n\n private Evaluator byTag() {\n // todo - these aren't dealing perfectly with case sensitivity. For case sensitive parsers, we should also make\n // the tag in the selector case-sensitive (and also attribute names). But for now, normalize (lower-case) for\n // consistency - both the selector and the element tag\n String tagName = normalize(tq.consumeElementSelector());\n Validate.notEmpty(tagName);\n\n // namespaces:\n if (tagName.startsWith(\"*|\")) { // namespaces: wildcard match equals(tagName) or ending in \":\"+tagName\n String plainTag = tagName.substring(2); // strip *|\n return new CombiningEvaluator.Or(\n new Evaluator.Tag(plainTag),\n new Evaluator.TagEndsWith(\":\" + plainTag)\n );\n } else if (tagName.endsWith(\"|*\")) { // ns|*\n String ns = tagName.substring(0, tagName.length() - 2) + \":\"; // strip |*, to ns:\n return new Evaluator.TagStartsWith(ns);\n } else if (tagName.contains(\"|\")) { // flip \"abc|def\" to \"abc:def\"\n tagName = tagName.replace(\"|\", \":\");\n }\n\n return new Evaluator.Tag(tagName);\n }\n\n private Evaluator byAttribute() {\n try (TokenQueue cq = new TokenQueue(tq.chompBalanced('[', ']'))) {\n return evaluatorForAttribute(cq);\n }\n }\n\n private Evaluator evaluatorForAttribute(TokenQueue cq) {\n String key = cq.consumeToAny(AttributeEvals); // eq, not, start, end, contain, match, (no val)\n key = normalize(key);\n Validate.notEmpty(key);\n Validate.isFalse(key.equals(\"abs:\"), \"Absolute attribute key must have a name\");\n cq.consumeWhitespace();\n final Evaluator eval;\n\n if (cq.isEmpty()) {\n if (key.startsWith(\"^\"))\n eval = new Evaluator.AttributeStarting(key.substring(1));\n else if (key.equals(\"*\")) // any attribute\n eval = new Evaluator.AttributeStarting(\"\");\n else\n eval = new Evaluator.Attribute(key);\n } else {\n if (cq.matchChomp('='))\n eval = new Evaluator.AttributeWithValue(key, cq.remainder());\n else if (cq.matchChomp(\"!=\"))\n eval = new Evaluator.AttributeWithValueNot(key, cq.remainder());\n else if (cq.matchChomp(\"^=\"))\n eval = new Evaluator.AttributeWithValueStarting(key, cq.remainder());\n else if (cq.matchChomp(\"$=\"))\n eval = new Evaluator.AttributeWithValueEnding(key, cq.remainder());\n else if (cq.matchChomp(\"*=\"))\n eval = new Evaluator.AttributeWithValueContaining(key, cq.remainder());\n else if (cq.matchChomp(\"~=\"))\n eval = new Evaluator.AttributeWithValueMatching(key, Regex.compile(cq.remainder()));\n else\n throw new Selector.SelectorParseException(\n \"Could not parse attribute query '%s': unexpected token at '%s'\", query, cq.remainder());\n }\n return eval;\n }\n\n //pseudo selectors :first-child, :last-child, :nth-child, ...\n private static final Pattern NthStepOffset = Pattern.compile(\"(([+-])?(\\\\d+)?)n(\\\\s*([+-])?\\\\s*\\\\d+)?\", Pattern.CASE_INSENSITIVE);\n private static final Pattern NthOffset = Pattern.compile(\"([+-])?(\\\\d+)\");\n\n private Evaluator cssNthChild(boolean last, boolean ofType) {\n String arg = normalize(consumeParens()); // arg is like \"odd\", or \"-n+2\", within nth-child(odd)\n final int step, offset;\n if (\"odd\".equals(arg)) {\n step = 2;\n offset = 1;\n } else if (\"even\".equals(arg)) {\n step = 2;\n offset = 0;\n } else {\n Matcher stepOffsetM, stepM;\n if ((stepOffsetM = NthStepOffset.matcher(arg)).matches()) {\n if (stepOffsetM.group(3) != null) // has digits, like 3n+2 or -3n+2\n step = Integer.parseInt(stepOffsetM.group(1).replaceFirst(\"^\\\\+\", \"\"));\n else // no digits, might be like n+2, or -n+2. if group(2) == \"-\", it’s -1;\n step = \"-\".equals(stepOffsetM.group(2)) ? -1 : 1;\n offset =\n stepOffsetM.group(4) != null ? Integer.parseInt(stepOffsetM.group(4).replaceFirst(\"^\\\\+\", \"\")) : 0;\n } else if ((stepM = NthOffset.matcher(arg)).matches()) {\n step = 0;\n offset = Integer.parseInt(stepM.group().replaceFirst(\"^\\\\+\", \"\"));\n } else {\n throw new Selector.SelectorParseException(\"Could not parse nth-index '%s': unexpected format\", arg);\n }\n }\n\n return ofType\n ? (last ? new Evaluator.IsNthLastOfType(step, offset) : new Evaluator.IsNthOfType(step, offset))\n : (last ? new Evaluator.IsNthLastChild(step, offset) : new Evaluator.IsNthChild(step, offset));\n }\n\n private String consumeParens() {\n return tq.chompBalanced('(', ')');\n }\n\n private int consumeIndex() {\n String index = consumeParens().trim();\n Validate.isTrue(StringUtil.isNumeric(index), \"Index must be numeric\");\n return Integer.parseInt(index);\n }\n\n // pseudo selector :has(el)\n private Evaluator has() {\n return parseNested(StructuralEvaluator.Has::new, \":has() must have a selector\");\n }\n\n // pseudo selector :is()\n private Evaluator is() {\n return parseNested(StructuralEvaluator.Is::new, \":is() must have a selector\");\n }\n\n private Evaluator parseNested(Function func, String err) {\n Validate.isTrue(tq.matchChomp('('), err);\n Evaluator eval = parseSelectorGroup();\n Validate.isTrue(tq.matchChomp(')'), err);\n return func.apply(eval);\n }\n\n // pseudo selector :contains(text), containsOwn(text)\n private Evaluator contains(boolean own) {\n String query = own ? \":containsOwn\" : \":contains\";\n String searchText = TokenQueue.unescape(consumeParens());\n Validate.notEmpty(searchText, query + \"(text) query must not be empty\");\n\n if (inNodeContext)\n return new NodeEvaluator.ContainsValue(searchText);\n\n return own\n ? new Evaluator.ContainsOwnText(searchText)\n : new Evaluator.ContainsText(searchText);\n }\n\n private Evaluator containsWholeText(boolean own) {\n String query = own ? \":containsWholeOwnText\" : \":containsWholeText\";\n String searchText = TokenQueue.unescape(consumeParens());\n Validate.notEmpty(searchText, query + \"(text) query must not be empty\");\n return own\n ? new Evaluator.ContainsWholeOwnText(searchText)\n : new Evaluator.ContainsWholeText(searchText);\n }\n\n // pseudo selector :containsData(data)\n private Evaluator containsData() {\n String searchText = TokenQueue.unescape(consumeParens());\n Validate.notEmpty(searchText, \":containsData(text) query must not be empty\");\n return new Evaluator.ContainsData(searchText);\n }\n\n // :matches(regex), matchesOwn(regex)\n private Evaluator matches(boolean own) {\n String query = own ? \":matchesOwn\" : \":matches\";\n String regex = consumeParens(); // don't unescape, as regex bits will be escaped\n Validate.notEmpty(regex, query + \"(regex) query must not be empty\");\n Regex pattern = Regex.compile(regex);\n\n if (inNodeContext)\n return new NodeEvaluator.MatchesValue(pattern);\n\n return own\n ? new Evaluator.MatchesOwn(pattern)\n : new Evaluator.Matches(pattern);\n }\n\n // :matches(regex), matchesOwn(regex)\n private Evaluator matchesWholeText(boolean own) {\n String query = own ? \":matchesWholeOwnText\" : \":matchesWholeText\";\n String regex = consumeParens(); // don't unescape, as regex bits will be escaped\n Validate.notEmpty(regex, query + \"(regex) query must not be empty\");\n\n Regex pattern = Regex.compile(regex);\n return own\n ? new Evaluator.MatchesWholeOwnText(pattern)\n : new Evaluator.MatchesWholeText(pattern);\n }\n\n // :not(selector)\n private Evaluator not() {\n String subQuery = consumeParens();\n Validate.notEmpty(subQuery, \":not(selector) subselect must not be empty\");\n\n return new StructuralEvaluator.Not(parse(subQuery));\n }\n\n @Override\n public String toString() {\n return query;\n }\n\n @Override\n public void close() {\n tq.close();\n }\n}\n"
},
{
"path": "src/main/java/org/jsoup/select/Selector.java",
"content": "package org.jsoup.select;\n\nimport org.jsoup.helper.Validate;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.nodes.Node;\nimport org.jsoup.parser.TokenQueue;\nimport org.jspecify.annotations.Nullable;\n\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.stream.Stream;\n\n/**\n CSS element selector, that finds elements matching a query.\n\n Selector syntax
\n \n A selector is a chain of simple selectors, separated by combinators. Selectors are case-insensitive (including\n against elements, attributes, and attribute values).\n
\n \n The universal selector {@code *} is implicit when no element selector is supplied (i.e. {@code .header} and\n {@code *.header} are equivalent).\n
\n\n You can easily test different selectors using the Try jsoup online playground.\n\n \n\n
\n | Pattern | Matches | Example |
|---|
\n * | any element | * |
\n tag | elements with the given tag name | div |
\n *|E | elements of type E in any namespace (including non-namespaced) | *|name finds <dc:name> and <name> elements |
\n ns|E | elements of type E in the namespace ns | dc|name finds <dc:name> elements |
\n ns|* | all elements in the namespace ns | dc|* finds <dc:p> and <dc:img>elements |
\n #id | elements with attribute ID of \"id\" | div#wrap, #logo |
\n .class | elements with a class name of \"class\" | div.left, .result |
\n [attr] | elements with an attribute named \"attr\" (with any value) | a[href], [title] |
\n [^attrPrefix] | elements with an attribute name starting with \"attrPrefix\". Use to find elements with HTML5 datasets | [^data-], div[^data-] |
\n [attr=val] | elements with an attribute named \"attr\", and value equal to \"val\" | img[width=500], a[rel=nofollow] |
\n [attr="val"] | elements with an attribute named \"attr\", and value equal to \"val\" | span[hello=\"Cleveland\"][goodbye=\"Columbus\"], a[rel="nofollow"] |
\n [attr^=valPrefix] | elements with an attribute named \"attr\", and value starting with \"valPrefix\" | a[href^=http:] |
\n [attr$=valSuffix] | elements with an attribute named \"attr\", and value ending with \"valSuffix\" | img[src$=.png] |
\n [attr*=valContaining] | elements with an attribute named \"attr\", and value containing \"valContaining\" | a[href*=/search/] |
\n [attr~=regex] | elements with an attribute named \"attr\", and value matching the regular expression | img[src~=(?i)\\\\.(png|jpe?g)] |
\n [*] | elements with any attribute | p[*] finds p elements that have at least one attribute; p:not([*]) finds those with no attributes |
\n | The above may be combined in any order | div.header[title] |
\n\n Combinators |
\n E F | an F element descended from an E element | div a, .logo h1 |
\n E {@literal >} F | an F direct child of E | ol {@literal >} li |
\n E + F | an F element immediately preceded by sibling E | li + li, div.head + div |
\n E ~ F | an F element preceded by sibling E | h1 ~ p |
\n E, F, G | all matching elements E, F, or G | a[href], div, h3 |
\n\n Pseudo selectors |
\n :lt(n) | elements whose sibling index is less than n | td:lt(3) finds the first 3 cells of each row |
\n :gt(n) | elements whose sibling index is greater than n | td:gt(1) finds cells after skipping the first two |
\n :eq(n) | elements whose sibling index is equal to n | td:eq(0) finds the first cell of each row |
\n :has(selector) | elements that contains at least one element matching the selector | div:has(p) finds divs that contain p elements.
div:has(> a) selects div elements that have at least one direct child a element.
section:has(h1, h2) finds section elements that contain a h1 or a h2 element |
\n :is(selector list) | elements that match any of the selectors in the selector list | :is(h1, h2, h3, h4, h5, h6) finds any heading element.
:is(section, article) > :is(h1, h2) finds a h1 or h2 that is a direct child of a section or an article |
\n :not(selector) | elements that do not match the selector. See also {@link Elements#not(String)} | div:not(.logo) finds all divs that do not have the \"logo\" class.div:not(:has(div)) finds divs that do not contain divs.
|
\n :contains(text) | elements that contains the specified text. The search is case insensitive. The text may appear in the found element, or any of its descendants. The text is whitespace normalized. To find content that includes parentheses, escape those with a {@code \\}. | p:contains(jsoup) finds p elements containing the text \"jsoup\".{@code p:contains(hello \\(there\\) finds p elements containing the text \"Hello (There)\"} |
\n :containsOwn(text) | elements that directly contain the specified text. The search is case insensitive. The text must appear in the found element, not any of its descendants. | p:containsOwn(jsoup) finds p elements with own text \"jsoup\". |
\n :containsData(data) | elements that contains the specified data. The contents of {@code script} and {@code style} elements, and {@code comment} nodes (etc) are considered data nodes, not text nodes. The search is case insensitive. The data may appear in the found element, or any of its descendants. | script:contains(jsoup) finds script elements containing the data \"jsoup\". |
\n :containsWholeText(text) | elements that contains the specified non-normalized text. The search is case sensitive, and will match exactly against spaces and newlines found in the original input. The text may appear in the found element, or any of its descendants. To find content that includes parentheses, escape those with a {@code \\}. | p:containsWholeText(jsoup\\nThe Java HTML Parser) finds p elements containing the text \"jsoup\\nThe Java HTML Parser\" (and not other variations of whitespace or casing, as :contains() would. Note that {@code br} elements are presented as a newline. |
\n :containsWholeOwnText(text) | elements that directly contain the specified non-normalized text. The search is case sensitive, and will match exactly against spaces and newlines found in the original input. The text may appear in the found element, but not in its descendants. To find content that includes parentheses, escape those with a {@code \\}. | p:containsWholeOwnText(jsoup\\nThe Java HTML Parser) finds p elements directly containing the text \"jsoup\\nThe Java HTML Parser\" (and not other variations of whitespace or casing, as :contains() would. Note that {@code br} elements are presented as a newline. |
\n :matches(regex) | elements containing whitespace normalized text that matches the specified regular expression. The text may appear in the found element, or any of its descendants. | td:matches(\\\\d+) finds table cells containing digits. div:matches((?i)login) finds divs containing the text, case insensitively. |
\n :matchesWholeText(regex) | elements containing non-normalized whole text that matches the specified regular expression. The text may appear in the found element, or any of its descendants. | td:matchesWholeText(\\\\s{2,}) finds table cells a run of at least two space characters. |
\n :matchesWholeOwnText(regex) | elements whose own non-normalized whole text matches the specified regular expression. The text must appear in the found element, not any of its descendants. | td:matchesWholeOwnText(\\n\\\\d+) finds table cells directly containing digits following a neewline. |
\n | The above may be combined in any order and with other selectors | .light:contains(name):eq(0) |
\n :matchText | treats text nodes as elements, and so allows you to match against and select text nodes. Note that using this selector will modify the DOM, so you may want to {@code clone} your document before using. Deprecated. This selector is deprecated and will be removed in a future version. Migrate to ::textnode using the Element#selectNodes() method instead. | {@code p:matchText:firstChild} with input {@code One
Two } will return one {@link org.jsoup.nodes.PseudoTextElement} with text \"{@code One}\". |
\n\n Structural pseudo selectors |
\n :root | The element that is the root of the document. In HTML, this is the html element | :root |
\n :nth-child(an+b) | elements that have an+b-1 siblings before it in the document tree, for any positive integer or zero value of n, and has a parent element. For values of a and b greater than zero, this effectively divides the element's children into groups of a elements (the last group taking the remainder), and selecting the bth element of each group. For example, this allows the selectors to address every other row in a table, and could be used to alternate the color of paragraph text in a cycle of four. The a and b values must be integers (positive, negative, or zero). The index of the first child of an element is 1. \n Additionally, :nth-child() supports odd and even as arguments. odd is the same as 2n+1, and even is the same as 2n. | tr:nth-child(2n+1) finds every odd row of a table. :nth-child(10n-1) the 9th, 19th, 29th, etc, element. li:nth-child(5) the 5h li |
\n :nth-last-child(an+b) | elements that have an+b-1 siblings after it in the document tree. Otherwise like :nth-child() | tr:nth-last-child(-n+2) the last two rows of a table |
\n :nth-of-type(an+b) | pseudo-class notation represents an element that has an+b-1 siblings with the same expanded element name before it in the document tree, for any zero or positive integer value of n, and has a parent element | img:nth-of-type(2n+1) |
\n :nth-last-of-type(an+b) | pseudo-class notation represents an element that has an+b-1 siblings with the same expanded element name after it in the document tree, for any zero or positive integer value of n, and has a parent element | img:nth-last-of-type(2n+1) |
\n :first-child | elements that are the first child of some other element. | div {@literal >} p:first-child |
\n :last-child | elements that are the last child of some other element. | ol {@literal >} li:last-child |
\n :first-of-type | elements that are the first sibling of its type in the list of children of its parent element | dl dt:first-of-type |
\n :last-of-type | elements that are the last sibling of its type in the list of children of its parent element | tr {@literal >} td:last-of-type |
\n :only-child | elements that have a parent element and whose parent element have no other element children | |
\n :only-of-type | an element that has a parent element and whose parent element has no other element children with the same expanded element name | |
\n :empty | elements that contain no child elements or nodes, with the exception of blank text nodes, comments, XML declarations, and doctype declarations. In other words, it matches elements that are effectively empty of meaningful content. | li:not(:empty) |
\n\n Node pseudo selectors |
\n These selectors enable matching specific leaf nodes, including Comments, TextNodes. When used with {@link Element#select(String)}, these can be used with structural selectors such as :has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@Element#selectNodes(String)}. |
\n | ::node | Matches any node | |
\n | ::leafnode | Matches any leaf-node (this is, a Node which is not an Element) | |
\n | ::comment | Matches a Comment node | |
\n | ::text | Matches a TextNode | |
\n | ::data | Matches a DataNode (e.g. the content of a script or a style element) | |
\n | ::cdata | Matches a CDataNode (which are only present in XML) | |
\n | ::node:contains(text) | Matches a node that has a (normalized, case-insensitive) value containing text. | ::comment:contains(foo bar) |
\n | ::node:matches(regex) | Matches a node that has a value matching the regex. | ::comment:matches(\\\\d+) |
\n | ::node:blank | Matches a node that has either no value, or a value of only whitespace. | ::comment:not(:blank) |
\n
\n\n A word on using regular expressions in these selectors: depending on the content of the regex, you will need to quote the pattern using Pattern.quote(\"regex\") for it to parse correctly through both the selector parser and the regex parser. E.g. String query = \"div:matches(\" + Pattern.quote(regex) + \");\".
\n Escaping special characters: to match a tag, ID, or other selector that does not follow the regular CSS syntax, the query must be escaped with the \\ character. For example, to match by ID {@code
}, use {@code document.select(\"#i\\\\.d\")}.
\n\n @see Element#select(String css)\n @see Element#selectFirst(String css)\n @see Element#select(Evaluator eval)\n @see Element#selectNodes(String css)\n @see Element#selectNodes(String css, Class nodeType)\n @see Elements#select(String css)\n @see Element#selectXpath(String xpath) */\npublic class Selector {\n // not instantiable\n private Selector() {}\n\n /**\n Find Elements matching the CSS query.\n\n @param query CSS selector\n @param root root element to descend into\n @return matching elements, empty if none\n @throws Selector.SelectorParseException (unchecked) on an invalid CSS query.\n */\n public static Elements select(String query, Element root) {\n Validate.notEmpty(query);\n return select(evaluatorOf(query), root);\n }\n\n /**\n Find Elements matching the Evaluator.\n\n @param evaluator CSS Evaluator\n @param root root (context) element to start from\n @return matching elements, empty if none\n */\n public static Elements select(Evaluator evaluator, Element root) {\n Validate.notNull(evaluator);\n Validate.notNull(root);\n return Collector.collect(evaluator, root);\n }\n\n /**\n Finds a Stream of elements matching the CSS query.\n\n @param query CSS selector\n @param root root element to descend into\n @return a Stream of matching elements, empty if none\n @throws Selector.SelectorParseException (unchecked) on an invalid CSS query.\n @since 1.19.1\n */\n public static Stream selectStream(String query, Element root) {\n Validate.notEmpty(query);\n return selectStream(evaluatorOf(query), root);\n }\n\n /**\n Finds a Stream of elements matching the evaluator.\n\n @param evaluator CSS selector\n @param root root element to descend into\n @return matching elements, empty if none\n @since 1.19.1\n */\n public static Stream selectStream(Evaluator evaluator, Element root) {\n Validate.notNull(evaluator);\n Validate.notNull(root);\n return Collector.stream(evaluator, root);\n }\n\n /**\n Find elements matching the query, across multiple roots. Elements will be deduplicated (in the case of\n overlapping hierarchies).\n\n @param query CSS selector\n @param roots root elements to descend into\n @return matching elements, empty if none\n */\n public static Elements select(String query, Iterable roots) {\n Validate.notEmpty(query);\n Validate.notNull(roots);\n Evaluator evaluator = evaluatorOf(query);\n Elements elements = new Elements();\n HashSet seenElements = new HashSet<>(); // dedupe elements by identity, as .equals is ==\n\n for (Element root : roots) {\n selectStream(evaluator, root)\n .filter(seenElements::add)\n .forEach(elements::add);\n }\n\n return elements;\n }\n\n // exclude set. package open so that Elements can implement .not() selector.\n static Elements filterOut(Collection elements, Collection outs) {\n Elements output = new Elements();\n for (Element el : elements) {\n boolean found = false;\n for (Element out : outs) {\n if (el.equals(out)) {\n found = true;\n break;\n }\n }\n if (!found)\n output.add(el);\n }\n return output;\n }\n\n /**\n Find the first Element that matches the query.\n\n @param cssQuery CSS selector\n @param root root element to descend into\n @return the matching element, or null if none.\n */\n public static @Nullable Element selectFirst(String cssQuery, Element root) {\n Validate.notEmpty(cssQuery);\n return Collector.findFirst(evaluatorOf(cssQuery), root);\n }\n\n /**\n Find the first element matching the query, across multiple roots.\n\n @param cssQuery CSS selector\n @param roots root elements to descend into\n @return the first matching element, or {@code null} if none\n @since 1.19.1\n */\n public static @Nullable Element selectFirst(String cssQuery, Iterable roots) {\n Validate.notEmpty(cssQuery);\n Validate.notNull(roots);\n Evaluator evaluator = evaluatorOf(cssQuery);\n\n for (Element root : roots) {\n Element first = Collector.findFirst(evaluator, root);\n if (first != null) return first;\n }\n\n return null;\n }\n\n /**\n Given a CSS identifier (such as a tag, ID, or class), escape any CSS special characters that would otherwise not be\n valid in a selector.\n\n @see CSS Object Model, serialize an identifier\n @since 1.20.1\n */\n public static String escapeCssIdentifier(String in) {\n return TokenQueue.escapeCssIdentifier(in);\n }\n\n /**\n Consume a CSS identifier (ID or class) off the queue.\n Note: For backwards compatibility this method supports improperly formatted CSS identifiers, e.g. {@code 1} instead\n of {@code \\31}.
\n\n @return The unescaped identifier.\n @throws IllegalArgumentException if an invalid escape sequence was found.\n @see CSS Syntax Module Level 3, Consume an ident sequence\n @see CSS Syntax Module Level 3, ident-token\n @since 1.20.1\n */\n public static String unescapeCssIdentifier(String in) {\n try (TokenQueue tq = new TokenQueue(in)) {\n return tq.consumeCssIdentifier();\n }\n }\n\n /**\n Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to\n parse it once and reuse the Evaluator.\n\n @param css CSS query\n @return Evaluator\n @see Selector selector query syntax\n @throws Selector.SelectorParseException if the CSS query is invalid\n @since 1.21.1\n */\n public static Evaluator evaluatorOf(String css) {\n return QueryParser.parse(css);\n }\n\n public static class SelectorParseException extends IllegalStateException {\n public SelectorParseException(String msg) {\n super(msg);\n }\n\n public SelectorParseException(String msg, Object... msgArgs) {\n super(String.format(msg, msgArgs));\n }\n\n public SelectorParseException(Throwable cause, String msg, Object... msgArgs) {\n super(String.format(msg, msgArgs), cause);\n }\n }\n}\n"
},
{
"path": "src/main/java/org/jsoup/select/StructuralEvaluator.java",
"content": "package org.jsoup.select;\n\nimport org.jsoup.internal.SoftPool;\nimport org.jsoup.internal.StringUtil;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.nodes.LeafNode;\nimport org.jsoup.nodes.Node;\nimport org.jsoup.nodes.NodeIterator;\nimport org.jsoup.nodes.TextNode;\n\nimport java.util.ArrayList;\nimport java.util.Map;\nimport java.util.WeakHashMap;\n\n/**\n * Base structural evaluator.\n */\nabstract class StructuralEvaluator extends Evaluator {\n final Evaluator evaluator;\n boolean wantsNodes; // if the evaluator requested nodes, not just elements\n\n public StructuralEvaluator(Evaluator evaluator) {\n this.evaluator = evaluator;\n wantsNodes = evaluator.wantsNodes();\n }\n\n @Override\n boolean wantsNodes() {\n return wantsNodes;\n }\n\n // Memoize inner matches, to save repeated re-evaluations of parent, sibling etc.\n // root + element: Boolean matches. ThreadLocal in case the Evaluator is compiled then reused across multi threads\n final ThreadLocal>> threadMemo = ThreadLocal.withInitial(WeakHashMap::new);\n\n boolean memoMatches(final Element root, final Node node) {\n Map> rootMemo = threadMemo.get();\n Map memo = rootMemo.computeIfAbsent(root, r -> new WeakHashMap<>());\n return memo.computeIfAbsent(node, test -> evaluator.matches(root, test));\n }\n\n @Override protected void reset() {\n threadMemo.remove();\n evaluator.reset();\n super.reset();\n }\n\n @Override\n public boolean matches(Element root, Element element) {\n return evaluateMatch(root, element);\n }\n\n @Override\n boolean matches(Element root, LeafNode leafNode) {\n return evaluateMatch(root, leafNode);\n }\n\n abstract boolean evaluateMatch(Element root, Node node);\n\n static class Root extends Evaluator {\n @Override\n public boolean matches(Element root, Element element) {\n return root == element;\n }\n\n @Override protected int cost() {\n return 1;\n }\n\n @Override public String toString() {\n return \">\";\n }\n }\n\n static class Has extends StructuralEvaluator {\n static final SoftPool> NodeIterPool =\n new SoftPool<>(() -> new NodeIterator<>(new TextNode(\"\"), Node.class));\n // the element here is just a placeholder so this can be final - gets set in restart()\n\n private final boolean checkSiblings; // evaluating against siblings (or children)\n\n public Has(Evaluator evaluator) {\n super(evaluator);\n checkSiblings = evalWantsSiblings(evaluator);\n }\n\n @Override public boolean matches(Element root, Element element) {\n if (checkSiblings) { // evaluating against siblings\n for (Element sib = element.firstElementSibling(); sib != null; sib = sib.nextElementSibling()) {\n if (sib != element && evaluator.matches(element, sib)) { // don't match against self\n return true;\n }\n }\n }\n // otherwise we only want to match children (or below), and not the input element. And we want to minimize GCs so reusing the Iterator obj\n NodeIterator it = NodeIterPool.borrow();\n it.restart(element);\n try {\n while (it.hasNext()) {\n Node node = it.next();\n if (node == element) continue; // don't match self, only descendants\n if (evaluator.matches(element, node)) {\n return true;\n }\n }\n } finally {\n NodeIterPool.release(it);\n }\n return false;\n }\n\n @Override\n boolean evaluateMatch(Element root, Node node) {\n return false; // unused; :has(::comment)) goes via implicit root combinator\n }\n\n /* Test if the :has sub-clause wants sibling elements (vs nested elements) - will be a Combining eval */\n private static boolean evalWantsSiblings(Evaluator eval) {\n if (eval instanceof CombiningEvaluator) {\n CombiningEvaluator ce = (CombiningEvaluator) eval;\n for (Evaluator innerEval : ce.evaluators) {\n if (innerEval instanceof PreviousSibling || innerEval instanceof ImmediatePreviousSibling)\n return true;\n }\n }\n return false;\n }\n\n @Override protected int cost() {\n return 10 * evaluator.cost();\n }\n\n @Override\n public String toString() {\n return String.format(\":has(%s)\", evaluator);\n }\n }\n\n /** Implements the :is(sub-query) pseudo-selector */\n static class Is extends StructuralEvaluator {\n public Is(Evaluator evaluator) {\n super(evaluator);\n }\n\n @Override\n boolean evaluateMatch(Element root, Node node) {\n return evaluator.matches(root, node);\n }\n\n @Override protected int cost() {\n return 2 + evaluator.cost();\n }\n\n @Override\n public String toString() {\n return String.format(\":is(%s)\", evaluator);\n }\n }\n\n static class Not extends StructuralEvaluator {\n public Not(Evaluator evaluator) {\n super(evaluator);\n }\n\n @Override\n boolean evaluateMatch(Element root, Node node) {\n return !memoMatches(root, node);\n }\n\n @Override protected int cost() {\n return 2 + evaluator.cost();\n }\n\n @Override\n public String toString() {\n return String.format(\":not(%s)\", evaluator);\n }\n }\n\n /**\n Any Ancestor (i.e., ascending parent chain.).\n */\n static class Ancestor extends StructuralEvaluator {\n public Ancestor(Evaluator evaluator) {\n super(evaluator);\n }\n\n @Override\n boolean evaluateMatch(Element root, Node node) {\n if (root == node)\n return false;\n\n for (Node parent = node.parent(); parent != null; parent = parent.parent()) {\n if (memoMatches(root, parent))\n return true;\n if (parent == root)\n break;\n }\n return false;\n }\n\n @Override\n protected int cost() {\n return 8 * evaluator.cost(); // probably lower than has(), but still significant, depending on doc and el depth.\n }\n\n @Override\n public String toString() {\n return String.format(\"%s \", evaluator);\n }\n }\n\n /**\n Holds a list of evaluators for one > two > three immediate parent matches, and the final direct evaluator under\n test. To match, these are effectively ANDed together, starting from the last, matching up to the first.\n */\n static class ImmediateParentRun extends StructuralEvaluator {\n final ArrayList evaluators = new ArrayList<>();\n int cost = 2;\n\n public ImmediateParentRun(Evaluator evaluator) {\n super(evaluator);\n evaluators.add(evaluator);\n cost += evaluator.cost();\n }\n\n void add(Evaluator evaluator) {\n evaluators.add(evaluator);\n cost += evaluator.cost();\n wantsNodes |= evaluator.wantsNodes();\n }\n\n @Override boolean evaluateMatch(Element root, Node node) {\n if (node == root)\n return false; // cannot match as the second eval (first parent test) would be above the root\n\n for (int i = evaluators.size() -1; i >= 0; --i) {\n if (node == null)\n return false;\n Evaluator eval = evaluators.get(i);\n if (!eval.matches(root, node))\n return false;\n node = node.parent();\n }\n return true;\n }\n\n @Override protected int cost() {\n return cost;\n }\n\n @Override\n protected void reset() {\n for (Evaluator evaluator : evaluators) {\n evaluator.reset();\n }\n super.reset();\n }\n\n @Override\n public String toString() {\n return StringUtil.join(evaluators, \" > \");\n }\n }\n\n static class PreviousSibling extends StructuralEvaluator {\n public PreviousSibling(Evaluator evaluator) {\n super(evaluator);\n }\n\n // matches any previous sibling, so can be same in Element only or wantsNodes context\n @Override boolean evaluateMatch(Element root, Node node) {\n if (root == node) return false;\n\n for (Node sib = node.firstSibling(); sib != null; sib = sib.nextSibling()) {\n if (sib == node) break;\n if (memoMatches(root, sib)) return true;\n }\n\n return false;\n }\n\n @Override protected int cost() {\n return 3 * evaluator.cost();\n }\n\n @Override\n public String toString() {\n return String.format(\"%s ~ \", evaluator);\n }\n }\n\n static class ImmediatePreviousSibling extends StructuralEvaluator {\n public ImmediatePreviousSibling(Evaluator evaluator) {\n super(evaluator);\n }\n\n @Override boolean evaluateMatch(Element root, Node node) {\n if (root == node) return false;\n\n Node prev = wantsNodes ? node.previousSibling() : node.previousElementSibling();\n return prev != null && memoMatches(root, prev);\n }\n\n @Override protected int cost() {\n return 2 + evaluator.cost();\n }\n\n @Override\n public String toString() {\n return String.format(\"%s + \", evaluator);\n }\n }\n}\n"
},
{
"path": "src/main/java/org/jsoup/select/package-info.java",
"content": "/**\n Packages to support the CSS-style element selector.\n {@link org.jsoup.select.Selector Selector defines the query syntax.}\n */\n@NullMarked\npackage org.jsoup.select;\n\nimport org.jspecify.annotations.NullMarked;\n"
},
{
"path": "src/main/java11/module-info.java",
"content": "module org.jsoup {\n exports org.jsoup;\n exports org.jsoup.helper;\n exports org.jsoup.nodes;\n exports org.jsoup.parser;\n exports org.jsoup.safety;\n exports org.jsoup.select;\n\n requires transitive java.xml; // for org.w3c.dom out of W3CDom\n requires static org.jspecify; // nullability annotations\n requires static java.net.http; // HttpClient on Java 11; guarded\n}\n"
},
{
"path": "src/main/java11/org/jsoup/helper/HttpClientExecutor.java",
"content": "package org.jsoup.helper;\n\nimport org.jsoup.Connection;\nimport org.jspecify.annotations.Nullable;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.net.Proxy;\nimport java.net.ProxySelector;\nimport java.net.SocketAddress;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.http.HttpClient;\nimport java.net.http.HttpHeaders;\nimport java.net.http.HttpRequest;\nimport java.net.http.HttpResponse;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\nimport static org.jsoup.helper.HttpConnection.Response;\nimport static org.jsoup.helper.HttpConnection.Response.writePost;\n\n/**\n Executes requests using the HttpClient, for http/2 support. Enabled by default when available. To disable, set\n property {@code jsoup.useHttpClient} to {@code false}.\n */\nclass HttpClientExecutor extends RequestExecutor {\n // HttpClient expects proxy settings per client; we do per request, so held as a thread local. Can't do same for\n // auth because that callback is on a worker thread, so can only do auth per Connection. So we create a new client\n // if the authenticator is different between requests\n static ThreadLocal<@Nullable Proxy> perRequestProxy = new ThreadLocal<>();\n\n @Nullable\n HttpResponse hRes;\n\n public HttpClientExecutor(HttpConnection.Request request, HttpConnection.@Nullable Response previousResponse) {\n super(request, previousResponse);\n }\n\n /**\n Retrieve the HttpClient from the Connection, or create a new one. Allows for connection pooling of requests in the\n same Connection (session).\n */\n HttpClient client() {\n // we try to reuse the same Client across requests in a given Connection; but if the request's auth or ssl context have changed, we need to create a new client\n if (req.connection.client != null) {\n HttpClient client = (HttpClient) req.connection.client;\n boolean reuse = true;\n\n RequestAuthenticator prevAuth = req.connection.lastAuth;\n req.connection.lastAuth = req.authenticator;\n if (prevAuth != req.authenticator) // might both be null\n reuse = false;\n if (req.sslContext != null && !(client.sslContext() == req.sslContext)) // client returns default context if not otherwise set\n reuse = false;\n if (reuse) return client;\n }\n\n HttpClient.Builder builder = HttpClient.newBuilder();\n builder.followRedirects(HttpClient.Redirect.NEVER); // customized redirects\n builder.proxy(new ProxyWrap()); // thread local impl for per request; called on executing thread\n if (req.authenticator != null) builder.authenticator(new AuthenticationHandler(req.authenticator));\n if (req.sslContext != null) builder.sslContext(req.sslContext);\n\n HttpClient client = builder.build();\n req.connection.client = client;\n return client;\n }\n\n @Override\n HttpConnection.Response execute() throws IOException {\n try {\n HttpRequest.Builder reqBuilder =\n HttpRequest.newBuilder(req.url.toURI()).method(req.method.name(), requestBody(req));\n if (req.timeout() > 0) reqBuilder.timeout(\n Duration.ofMillis(req.timeout())); // infinite if unset (UrlConnection / jsoup uses 0 for same)\n CookieUtil.applyCookiesToRequest(req, reqBuilder::header);\n\n // headers:\n req.multiHeaders().forEach((key, values) -> {\n values.forEach(value -> reqBuilder.header(key, value));\n });\n\n if (req.proxy() != null) perRequestProxy.set(req.proxy()); // set up per request proxy\n HttpRequest hReq = reqBuilder.build();\n HttpClient client = client();\n hRes = client.send(hReq, HttpResponse.BodyHandlers.ofInputStream());\n HttpHeaders headers = hRes.headers();\n\n // set up the response\n Response res = new Response(req);\n res.executor = this;\n res.method = Connection.Method.valueOf(hRes.request().method());\n res.url = hRes.uri().toURL();\n res.statusCode = hRes.statusCode();\n res.statusMessage = StatusMessage(res.statusCode);\n res.contentType = headers.firstValue(\"content-type\").orElse(\"\");\n long length = headers.firstValueAsLong(\"content-length\").orElse(-1);\n res.contentLength = length < Integer.MAX_VALUE ? (int) length : -1;\n res.prepareResponse(headers.map(), prevRes);\n\n return res;\n } catch (IOException e) {\n safeClose();\n throw e;\n } catch (InterruptedException e) {\n safeClose();\n Thread.currentThread().interrupt();\n throw new IOException(e);\n } catch (URISyntaxException e) {\n throw new IllegalArgumentException(\"Malformed URL: \" + req.url, e);\n } finally {\n // detach per request proxy\n perRequestProxy.remove();\n }\n }\n\n /**\n As HTTP/2 no longer provides a server-set status message, and HttpClient doesn't parse it for 1.1, just provide minimal stock ones, for loggers.\n */\n static String StatusMessage(int statusCode) {\n if (statusCode < 400) return \"OK\";\n if (statusCode == 404) return \"Not Found\";\n return \"Error \" + statusCode;\n }\n\n @Override\n InputStream responseBody() throws IOException {\n if (hRes == null) throw new IllegalStateException(\"Not yet executed\");\n return hRes.body();\n }\n\n @Override\n void safeClose() {\n if (hRes != null) {\n InputStream body = hRes.body();\n if (body != null) {\n try {\n body.close();\n } catch (IOException ignored) {}\n }\n hRes = null;\n }\n }\n\n static HttpRequest.BodyPublisher requestBody(final HttpConnection.Request req) throws IOException {\n if (req.method.hasBody()) {\n ByteArrayOutputStream buf = new ByteArrayOutputStream();\n writePost(req, buf);\n return HttpRequest.BodyPublishers.ofByteArray(buf.toByteArray());\n } else {\n return HttpRequest.BodyPublishers.noBody();\n }\n }\n\n static class ProxyWrap extends ProxySelector {\n // empty list for no proxy:\n static final List NoProxy = new ArrayList<>(0);\n\n @Override\n public List select(URI uri) {\n Proxy proxy = perRequestProxy.get();\n if (proxy != null) {\n return Collections.singletonList(proxy);\n }\n ProxySelector defaultSelector = ProxySelector.getDefault();\n if (defaultSelector != null && defaultSelector != this) { // avoid recursion if we were set as default\n return defaultSelector.select(uri);\n }\n return NoProxy;\n }\n\n @Override\n public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {\n if (perRequestProxy.get() != null) {\n return; // no-op\n }\n ProxySelector defaultSelector = ProxySelector.getDefault();\n if (defaultSelector != null && defaultSelector != this) {\n defaultSelector.connectFailed(uri, sa, ioe);\n }\n }\n }\n}\n"
},
{
"path": "src/main/java11/org/jsoup/helper/RequestAuthHandler.java",
"content": "package org.jsoup.helper;\n\nimport java.net.HttpURLConnection;\nimport java.net.http.HttpClient;\n\n/**\n A per-request authentication shim, used in Java 9+.\n */\nclass RequestAuthHandler implements AuthenticationHandler.AuthShim {\n public RequestAuthHandler() {}\n\n @Override public void enable(RequestAuthenticator auth, Object connOrHttp) {\n AuthenticationHandler authenticator = new AuthenticationHandler(auth);\n\n // this is a bit ugly, but a simple way to support setting authentication on both urlconnection and httpclient without more multi-version shims\n if (connOrHttp instanceof HttpURLConnection) {\n HttpURLConnection conn = (HttpURLConnection) connOrHttp;\n conn.setAuthenticator(authenticator);\n } else if (connOrHttp instanceof HttpClient.Builder) {\n HttpClient.Builder builder = (HttpClient.Builder) connOrHttp;\n builder.authenticator(authenticator);\n } else {\n throw new IllegalArgumentException(\"Unsupported executor: \" + connOrHttp.getClass().getName());\n }\n }\n\n @Override public void remove() {\n // noop; would remove thread-local in Global Handler\n }\n\n @Override public AuthenticationHandler get(AuthenticationHandler helper) {\n // would get thread-local in Global Handler\n return helper;\n }\n}\n"
},
{
"path": "src/main/javadoc/overview.html",
"content": "\n\n\n jsoup Javadoc overview\n\n\njsoup: Java HTML parser that makes sense of real-world HTML soup.
\n\njsoup is a Java library for working with real-world HTML. It provides a very convenient API for fetching URLs\n and extracting and manipulating data, using the best of HTML5 DOM methods and CSS selectors.
\n\njsoup implements the WHATWG HTML specification, and parses HTML to the same DOM\n as modern browsers do.
\n\n\n - parse HTML from a URL, file, or string\n
- find and extract data, using DOM traversal or CSS selectors\n
- manipulate the HTML elements, attributes, and text\n
- clean user-submitted content against a safelist, to prevent XSS\n
- output tidy HTML\n
\n\njsoup is designed to deal with all varieties of HTML found in the wild; from pristine and validating,\n to invalid tag-soup; jsoup will create a sensible parse tree.
\n\nSee jsoup.org for downloads, documentation, and examples.
\n\n@author Jonathan Hedley\n\n\n\n"
},
{
"path": "src/main/resources/META-INF/proguard/org.jsoup_jsoup.pro",
"content": "-dontwarn com.google.re2j.**\n"
},
{
"path": "src/test/java/org/jsoup/JsoupTest.java",
"content": "package org.jsoup;\n\nimport org.jsoup.integration.ParseTest;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.IOException;\nimport java.nio.file.Path;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class JsoupTest {\n // Tests for the Jsoup class. Mostly for code coverage for methods that haven't been covered elsewhere already.\n\n @Test\n void parseWithPath() throws IOException {\n // parse(Path path, @Nullable String charsetName, String baseUri)\n Path path = ParseTest.getPath(\"/htmltests/medium.html\");\n Document doc = Jsoup.parse(path, \"UTF-8\", \"https://example.com/\");\n String title = \"Medium HTML\";\n assertEquals(title, doc.title());\n\n // parse(Path path)\n doc = Jsoup.parse(path);\n assertEquals(title, doc.title());\n\n // (Path path, @Nullable String charsetName, String baseUri, Parser parser)\n doc = Jsoup.parse(path, \"UTF-8\", \"https://example.com/\", Parser.htmlParser());\n assertEquals(title, doc.title());\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/MultiLocaleExtension.java",
"content": "package org.jsoup;\n\nimport org.junit.jupiter.api.extension.AfterEachCallback;\nimport org.junit.jupiter.api.extension.ExtendWith;\nimport org.junit.jupiter.api.extension.ExtensionContext;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.Arguments;\nimport org.junit.jupiter.params.provider.ArgumentsProvider;\nimport org.junit.jupiter.params.provider.ArgumentsSource;\n\nimport java.lang.annotation.*;\nimport java.util.Locale;\nimport java.util.stream.Stream;\n\npublic class MultiLocaleExtension implements AfterEachCallback, ArgumentsProvider {\n private final Locale defaultLocale = Locale.getDefault();\n\n @Override\n public void afterEach(ExtensionContext context) {\n Locale.setDefault(defaultLocale);\n }\n\n @Override\n public Stream provideArguments(ExtensionContext extensionContext) {\n return Stream.of(Arguments.of(Locale.ENGLISH), Arguments.arguments(new Locale(\"tr\")));\n }\n\n\n @Documented\n @Target(ElementType.METHOD)\n @Retention(RetentionPolicy.RUNTIME)\n @ArgumentsSource(MultiLocaleExtension.class)\n @ExtendWith(MultiLocaleExtension.class)\n @ParameterizedTest\n public @interface MultiLocaleTest {\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/SerializationExceptionTest.java",
"content": "package org.jsoup;\n\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class SerializationExceptionTest {\n @Test void constructors() {\n SerializationException e = new SerializationException(\"message\");\n assertEquals(\"message\", e.getMessage());\n assertNull(e.getCause());\n\n SerializationException e2 = new SerializationException(\"message\", new Exception(\"cause\"));\n assertEquals(\"message\", e2.getMessage());\n assertEquals(\"cause\", e2.getCause().getMessage());\n\n SerializationException e3 = new SerializationException();\n assertNull(e3.getMessage());\n assertNull(e3.getCause());\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/TextUtil.java",
"content": "package org.jsoup;\n\nimport java.util.regex.Pattern;\n\n/**\n Text utils to ease testing\n\n @author Jonathan Hedley, jonathan@hedley.net */\npublic class TextUtil {\n static Pattern stripper = Pattern.compile(\"\\\\r?\\\\n\\\\s*\");\n static Pattern stripLines = Pattern.compile(\"\\\\r?\\\\n?\");\n static Pattern spaceCollapse = Pattern.compile(\"\\\\s{2,}\");\n static Pattern tagSpaceCollapse = Pattern.compile(\">\\\\s+<\");\n static Pattern stripCRs = Pattern.compile(\"\\\\r*\");\n\n public static String stripNewlines(String text) {\n return stripper.matcher(text).replaceAll(\"\");\n }\n\n public static String normalizeSpaces(String text) {\n text = stripLines.matcher(text).replaceAll(\"\");\n text = stripper.matcher(text).replaceAll(\"\");\n text = spaceCollapse.matcher(text).replaceAll(\" \");\n text = tagSpaceCollapse.matcher(text).replaceAll(\"><\");\n return text;\n }\n\n public static String stripCRs(String text) {\n return stripCRs.matcher(text).replaceAll(\"\");\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/AuthenticationHandlerTest.java",
"content": "package org.jsoup.helper;\n\npublic class AuthenticationHandlerTest {\n public static final int MaxAttempts = AuthenticationHandler.MaxAttempts;\n\n // tests are in ConnectionTest, ProxyTest. This class just makes the MaxAttempts visible for test.\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/CookieUtilTest.java",
"content": "package org.jsoup.helper;\n\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass CookieUtilTest {\n\n @Test void parseCookie() {\n HttpConnection.Response res = new HttpConnection.Response();\n\n CookieUtil.parseCookie(\"foo=bar qux; Domain=.example.com; Path=/; Secure\", res);\n CookieUtil.parseCookie(\"bar=foo qux\", res);\n CookieUtil.parseCookie(\"=bar; Domain=.example.com; Path=/; Secure\", res);\n CookieUtil.parseCookie(\"; Domain=.example.com; Path=/\", res);\n CookieUtil.parseCookie(\"\", res);\n CookieUtil.parseCookie(null, res);\n\n assertEquals(3, res.cookies().size());\n assertEquals(\"bar qux\", res.cookies.get(\"foo\"));\n assertEquals(\"foo qux\", res.cookies.get(\"bar\"));\n assertEquals(\".example.com\", res.cookies.get(\"; Domain\")); // no actual cookie name or val\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/DataUtilTest.java",
"content": "package org.jsoup.helper;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.integration.ParseTest;\nimport org.jsoup.internal.ControllableInputStream;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.parser.Parser;\nimport org.jsoup.parser.StreamParser;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.*;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\n\nimport static org.jsoup.integration.ParseTest.getFile;\nimport static org.jsoup.integration.ParseTest.getPath;\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class DataUtilTest {\n @Test\n public void testCharset() {\n assertEquals(\"utf-8\", DataUtil.getCharsetFromContentType(\"text/html;charset=utf-8 \"));\n assertEquals(\"UTF-8\", DataUtil.getCharsetFromContentType(\"text/html; charset=UTF-8\"));\n assertEquals(\"ISO-8859-1\", DataUtil.getCharsetFromContentType(\"text/html; charset=ISO-8859-1\"));\n assertNull(DataUtil.getCharsetFromContentType(\"text/html\"));\n assertNull(DataUtil.getCharsetFromContentType(null));\n assertNull(DataUtil.getCharsetFromContentType(\"text/html;charset=Unknown\"));\n }\n\n @Test\n public void testQuotedCharset() {\n assertEquals(\"utf-8\", DataUtil.getCharsetFromContentType(\"text/html; charset=\\\"utf-8\\\"\"));\n assertEquals(\"UTF-8\", DataUtil.getCharsetFromContentType(\"text/html;charset=\\\"UTF-8\\\"\"));\n assertEquals(\"ISO-8859-1\", DataUtil.getCharsetFromContentType(\"text/html; charset=\\\"ISO-8859-1\\\"\"));\n assertNull(DataUtil.getCharsetFromContentType(\"text/html; charset=\\\"Unsupported\\\"\"));\n assertEquals(\"UTF-8\", DataUtil.getCharsetFromContentType(\"text/html; charset='UTF-8'\"));\n }\n\n private ControllableInputStream stream(String data) {\n return ControllableInputStream.wrap(new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)), 0);\n }\n\n private ControllableInputStream stream(String data, String charset) {\n return ControllableInputStream.wrap(new ByteArrayInputStream(data.getBytes(Charset.forName(charset))), 0);\n }\n\n @Test\n public void discardsSpuriousByteOrderMark() throws IOException {\n String html = \"\\uFEFFOneTwo\";\n Document doc = DataUtil.parseInputStream(stream(html), \"UTF-8\", \"http://foo.com/\", Parser.htmlParser());\n assertEquals(\"One\", doc.head().text());\n }\n\n @Test\n public void discardsSpuriousByteOrderMarkWhenNoCharsetSet() throws IOException {\n String html = \"\\uFEFFOneTwo\";\n Document doc = DataUtil.parseInputStream(stream(html), null, \"http://foo.com/\", Parser.htmlParser());\n assertEquals(\"One\", doc.head().text());\n assertEquals(\"UTF-8\", doc.outputSettings().charset().displayName());\n }\n\n @Test\n public void shouldNotThrowExceptionOnEmptyCharset() {\n assertNull(DataUtil.getCharsetFromContentType(\"text/html; charset=\"));\n assertNull(DataUtil.getCharsetFromContentType(\"text/html; charset=;\"));\n }\n\n @Test\n public void shouldSelectFirstCharsetOnWeirdMultileCharsetsInMetaTags() {\n assertEquals(\"ISO-8859-1\", DataUtil.getCharsetFromContentType(\"text/html; charset=ISO-8859-1, charset=1251\"));\n }\n\n @Test\n public void shouldCorrectCharsetForDuplicateCharsetString() {\n assertEquals(\"iso-8859-1\", DataUtil.getCharsetFromContentType(\"text/html; charset=charset=iso-8859-1\"));\n }\n\n @Test\n public void shouldReturnNullForIllegalCharsetNames() {\n assertNull(DataUtil.getCharsetFromContentType(\"text/html; charset=$HJKDF§$/(\"));\n }\n\n @Test\n public void generatesMimeBoundaries() {\n String m1 = DataUtil.mimeBoundary();\n String m2 = DataUtil.mimeBoundary();\n\n assertEquals(DataUtil.boundaryLength, m1.length());\n assertEquals(DataUtil.boundaryLength, m2.length());\n assertNotSame(m1, m2);\n }\n\n @Test\n public void wrongMetaCharsetFallback() throws IOException {\n String html = \"\";\n\n Document doc = DataUtil.parseInputStream(stream(html), null, \"http://example.com\", Parser.htmlParser());\n\n final String expected = \"\\n\" +\n \" \\n\" +\n \" \\n\" +\n \" \\n\" +\n \" \\n\" +\n \"\";\n\n assertEquals(expected, doc.toString());\n }\n\n @Test\n public void secondMetaElementWithContentTypeContainsCharsetParameter() throws Exception {\n String html = \"\" +\n \"\" +\n \"\" +\n \"한국어\";\n\n Document doc = DataUtil.parseInputStream(stream(html, \"euc-kr\"), null, \"http://example.com\", Parser.htmlParser());\n\n assertEquals(\"한국어\", doc.body().text());\n }\n\n @Test\n public void firstMetaElementWithCharsetShouldBeUsedForDecoding() throws Exception {\n String html = \"\" +\n \"\" +\n \"\" +\n \"Übergrößenträger\";\n\n Document doc = DataUtil.parseInputStream(stream(html, \"iso-8859-1\"), null, \"http://example.com\", Parser.htmlParser());\n\n assertEquals(\"Übergrößenträger\", doc.body().text());\n }\n\n @Test\n public void parseSequenceInputStream() throws IOException {\n // https://github.com/jhy/jsoup/pull/1671\n File in = getFile(\"/htmltests/medium.html\");\n String fileContent = new String(Files.readAllBytes(in.toPath()));\n int halfLength = fileContent.length() / 2;\n String firstPart = fileContent.substring(0, halfLength);\n String secondPart = fileContent.substring(halfLength);\n SequenceInputStream sequenceStream = new SequenceInputStream(\n stream(firstPart),\n stream(secondPart)\n );\n ControllableInputStream stream = ControllableInputStream.wrap(sequenceStream, 0);\n Document doc = DataUtil.parseInputStream(stream, null, \"\", Parser.htmlParser());\n assertEquals(fileContent, doc.outerHtml());\n }\n\n @Test\n public void supportsBOMinFiles() throws IOException {\n // test files from http://www.i18nl10n.com/korean/utftest/\n File in = getFile(\"/bomtests/bom_utf16be.html\");\n Document doc = Jsoup.parse(in, null, \"http://example.com\");\n assertTrue(doc.title().contains(\"UTF-16BE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf16le.html\");\n doc = Jsoup.parse(in, null, \"http://example.com\");\n assertTrue(doc.title().contains(\"UTF-16LE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf32be.html\");\n doc = Jsoup.parse(in, null, \"http://example.com\");\n assertTrue(doc.title().contains(\"UTF-32BE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf32le.html\");\n doc = Jsoup.parse(in, null, \"http://example.com\");\n assertTrue(doc.title().contains(\"UTF-32LE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n }\n\n @Test\n public void streamerSupportsBOMinFiles() throws IOException {\n // test files from http://www.i18nl10n.com/korean/utftest/\n Path in = getFile(\"/bomtests/bom_utf16be.html\").toPath();\n Parser parser = Parser.htmlParser();\n Document doc = DataUtil.streamParser(in, null, \"http://example.com\", parser).complete();\n assertTrue(doc.title().contains(\"UTF-16BE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf16le.html\").toPath();\n doc = DataUtil.streamParser(in, null, \"http://example.com\", parser).complete();\n assertTrue(doc.title().contains(\"UTF-16LE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf32be.html\").toPath();\n doc = DataUtil.streamParser(in, null, \"http://example.com\", parser).complete();\n assertTrue(doc.title().contains(\"UTF-32BE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n\n in = getFile(\"/bomtests/bom_utf32le.html\").toPath();\n doc = DataUtil.streamParser(in, null, \"http://example.com\", parser).complete();\n assertTrue(doc.title().contains(\"UTF-32LE\"));\n assertTrue(doc.text().contains(\"가각갂갃간갅\"));\n }\n\n @Test\n public void supportsUTF8BOM() throws IOException {\n File in = getFile(\"/bomtests/bom_utf8.html\");\n Document doc = Jsoup.parse(in, null, \"http://example.com\");\n assertEquals(\"OK\", doc.head().select(\"title\").text());\n }\n\n @Test\n public void noExtraNULLBytes() throws IOException {\n \tfinal byte[] b = \"üü
\".getBytes(StandardCharsets.UTF_8);\n\n \tDocument doc = Jsoup.parse(new ByteArrayInputStream(b), null, \"\");\n \tassertFalse( doc.outerHtml().contains(\"\\u0000\") );\n }\n\n @Test\n public void supportsZippedUTF8BOM() throws IOException {\n File in = getFile(\"/bomtests/bom_utf8.html.gz\");\n Document doc = Jsoup.parse(in, null, \"http://example.com\");\n assertEquals(\"OK\", doc.head().select(\"title\").text());\n assertEquals(\"There is a UTF8 BOM at the top (before the XML decl). If not read correctly, will look like a non-joining space.\", doc.body().text());\n }\n\n @Test\n public void streamerSupportsZippedUTF8BOM() throws IOException {\n Path in = getFile(\"/bomtests/bom_utf8.html.gz\").toPath();\n Document doc = DataUtil.streamParser(in, null, \"http://example.com\", Parser.htmlParser()).complete();\n assertEquals(\"OK\", doc.head().select(\"title\").text());\n assertEquals(\"There is a UTF8 BOM at the top (before the XML decl). If not read correctly, will look like a non-joining space.\", doc.body().text());\n }\n\n @Test\n public void supportsXmlCharsetDeclaration() throws IOException {\n String encoding = \"iso-8859-1\";\n InputStream soup = new ByteArrayInputStream((\n \"\" +\n \"\" +\n \"Hellö Wörld!\"\n ).getBytes(Charset.forName(encoding)));\n\n Document doc = Jsoup.parse(soup, null, \"\");\n assertEquals(\"Hellö Wörld!\", doc.body().text());\n }\n\n\n @Test\n public void loadsGzipFile() throws IOException {\n File in = getFile(\"/htmltests/gzip.html.gz\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"Gzip test\", doc.title());\n assertEquals(\"This is a gzipped HTML file.\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void loadsGzipPath() throws IOException {\n Path in = getPath(\"/htmltests/gzip.html.gz\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"Gzip test\", doc.title());\n assertEquals(\"This is a gzipped HTML file.\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void loadsZGzipFile() throws IOException {\n // compressed on win, with z suffix\n File in = getFile(\"/htmltests/gzip.html.z\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"Gzip test\", doc.title());\n assertEquals(\"This is a gzipped HTML file.\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void loadsZGzipPath() throws IOException {\n // compressed on win, with z suffix\n Path in = getPath(\"/htmltests/gzip.html.z\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"Gzip test\", doc.title());\n assertEquals(\"This is a gzipped HTML file.\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void handlesFakeGzipFile() throws IOException {\n File in = getFile(\"/htmltests/fake-gzip.html.gz\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"This is not gzipped\", doc.title());\n assertEquals(\"And should still be readable.\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void handlesFakeGzipPath() throws IOException {\n Path in = getPath(\"/htmltests/fake-gzip.html.gz\");\n Document doc = Jsoup.parse(in, null);\n assertEquals(\"This is not gzipped\", doc.title());\n assertEquals(\"And should still be readable.\", doc.selectFirst(\"p\").text());\n }\n\n // an input stream to give a range of output sizes, that changes on each read\n static class VaryingReadInputStream extends InputStream {\n final InputStream in;\n int stride = 0;\n\n VaryingReadInputStream(InputStream in) {\n this.in = in;\n }\n\n public int read() throws IOException {\n return in.read();\n }\n\n public int read(byte[] b) throws IOException {\n return in.read(b, 0, Math.min(b.length, ++stride));\n }\n\n public int read(byte[] b, int off, int len) throws IOException {\n return in.read(b, off, Math.min(len, ++stride));\n }\n }\n\n @Test\n void handlesChunkedInputStream() throws IOException {\n File inputFile = ParseTest.getFile(\"/htmltests/large.html\");\n String input = ParseTest.getFileAsString(inputFile);\n VaryingReadInputStream stream = new VaryingReadInputStream(ParseTest.inputStreamFrom(input));\n\n Document expected = Jsoup.parse(input, \"https://example.com\");\n Document doc = Jsoup.parse(stream, null, \"https://example.com\");\n assertTrue(doc.hasSameValue(expected));\n }\n\n @Test\n void handlesUnlimitedRead() throws IOException {\n File inputFile = ParseTest.getFile(\"/htmltests/large.html\");\n String input = ParseTest.getFileAsString(inputFile);\n VaryingReadInputStream stream = new VaryingReadInputStream(ParseTest.inputStreamFrom(input));\n\n ByteBuffer byteBuffer = DataUtil.readToByteBuffer(stream, 0);\n String read = new String(byteBuffer.array(), 0, byteBuffer.limit(), StandardCharsets.UTF_8);\n\n assertEquals(input, read);\n }\n\n @Test void controllableInputStreamAllowsNull() throws IOException {\n ControllableInputStream is = ControllableInputStream.wrap(null, 0);\n assertNotNull(is);\n assertTrue(is.baseReadFully());\n is.close();\n }\n\n @Test void streamParserSurrogateAcrossBuffer() throws IOException {\n // https://github.com/jhy/jsoup/issues/2353\n try (StreamParser parser = DataUtil.streamParser(ParseTest.getPath(\"/fuzztests/2353.html.gz\"), DataUtil.UTF_8, \"\", Parser.htmlParser())) {\n Document doc = parser.complete();\n String html = doc.html();\n assertTrue(html.contains(\"Read-Fully!\"));\n }\n }\n\n @Test void parseSurrogateAcrossBuffer() throws IOException {\n Document doc = Jsoup.parse(ParseTest.getPath(\"/fuzztests/2353.html.gz\"));\n assertTrue(doc.html().contains(\"Read-Fully!\"));\n }\n\n @Test\n void charsetSniffingCanReuseTruncatedPreParse() throws IOException {\n // #2448: when available() reports buffered bytes after the first read, the sniffed pre-parse may be reused while capped, leading to truncation\n\n StringBuilder sb = new StringBuilder();\n sb.append(\"t\");\n while (sb.length() < 6200) {\n sb.append(\"0123456789 abcdefghijklmnopqrstuvwxyz\\n\");\n }\n sb.append(\"list
\");\n String html = sb.toString();\n\n\n byte[] bytes = html.getBytes(StandardCharsets.UTF_8);\n ControllableInputStream in = ControllableInputStream.wrap(new BufferedOnceAvailableStream(bytes), 0);\n\n DataUtil.CharsetDoc charsetDoc = DataUtil.detectCharset(in, null, \"http://example.com/\", Parser.htmlParser());\n Document doc = DataUtil.parseInputStream(charsetDoc, \"http://example.com/\", Parser.htmlParser());\n\n assertNotNull(doc.selectFirst(\"hr\"), \"hr should survive the sniff + full parse\");\n }\n\n // delivers all bytes in the first read, then signals available()>0 once to trigger a second read and baseReadFully=true\n static final class BufferedOnceAvailableStream extends InputStream {\n private final byte[] data;\n private int pos = 0;\n private boolean extraSignal = true;\n\n BufferedOnceAvailableStream(byte[] data) {\n this.data = data;\n }\n\n @Override\n public int read(byte[] b, int off, int len) {\n if (pos >= data.length) return -1;\n int take = Math.min(len, data.length - pos);\n System.arraycopy(data, pos, b, off, take);\n pos += take;\n return take;\n }\n\n @Override\n public int read() {\n return pos < data.length ? (data[pos++] & 0xff) : -1;\n }\n\n @Override\n public int available() {\n if (pos < data.length)\n return data.length - pos;\n if (extraSignal) {\n extraSignal = false;\n return 1; // nudge SimpleBufferedInput.fill() to try another read\n }\n return 0;\n }\n }\n\n @Test\n void charsetSniffingIgnoresAdvisoryAvailableIOException() throws IOException {\n // https://github.com/jhy/jsoup/issues/2474\n // JDK 8's HttpURLConnection stream may throw from available() once the peer has closed the socket;\n // that advisory failure does not mean we can't still consume bytes already buffered or read to clean EOF.\n String html = \"OneTwo\";\n byte[] bytes = html.getBytes(StandardCharsets.UTF_8);\n InputStream stream = new FilterInputStream(new ByteArrayInputStream(bytes)) {\n @Override\n public int available() throws IOException {\n throw new IOException(\"Stream closed.\");\n }\n };\n ControllableInputStream in = ControllableInputStream.wrap(stream, 0);\n\n Document doc = DataUtil.parseInputStream(in, null, \"http://example.com/\", Parser.htmlParser());\n\n assertEquals(\"One\", doc.title());\n assertEquals(\"Two\", doc.body().text());\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/HttpConnectionTest.java",
"content": "package org.jsoup.helper;\n\nimport org.jsoup.Connection;\nimport org.jsoup.Jsoup;\nimport org.jsoup.MultiLocaleExtension.MultiLocaleTest;\nimport org.jsoup.integration.ParseTest;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.ValueSource;\n\nimport java.io.IOException;\nimport java.net.Authenticator;\nimport java.net.MalformedURLException;\nimport java.net.PasswordAuthentication;\nimport java.net.URL;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\n\nimport static org.jsoup.helper.HttpConnection.Response.fixHeaderEncoding;\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class HttpConnectionTest {\n /* most actual network http connection tests are in integration */\n\n @Test public void canCreateEmptyConnection() {\n HttpConnection con = new HttpConnection();\n assertEquals(Connection.Method.GET, con.request().method());\n assertThrows(IllegalArgumentException.class, () -> {\n URL url = con.request().url();\n });\n }\n\n @Test public void throwsExceptionOnResponseWithoutExecute() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection con = HttpConnection.connect(\"http://example.com\");\n con.response();\n });\n }\n\n @Test public void throwsExceptionOnParseWithoutExecute() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection con = HttpConnection.connect(\"http://example.com\");\n con.response().parse();\n });\n }\n\n @Test public void throwsExceptionOnBodyWithoutExecute() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection con = HttpConnection.connect(\"http://example.com\");\n con.response().body();\n });\n }\n\n @Test public void throwsExceptionOnBodyAsBytesWithoutExecute() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection con = HttpConnection.connect(\"http://example.com\");\n con.response().bodyAsBytes();\n });\n }\n\n @MultiLocaleTest\n public void caseInsensitiveHeaders(Locale locale) {\n Locale.setDefault(locale);\n\n Connection.Response res = new HttpConnection.Response();\n res.header(\"Accept-Encoding\", \"gzip\");\n res.header(\"content-type\", \"text/html\");\n res.header(\"refErrer\", \"http://example.com\");\n\n assertTrue(res.hasHeader(\"Accept-Encoding\"));\n assertTrue(res.hasHeader(\"accept-encoding\"));\n assertTrue(res.hasHeader(\"accept-Encoding\"));\n assertTrue(res.hasHeader(\"ACCEPT-ENCODING\"));\n\n assertEquals(\"gzip\", res.header(\"accept-Encoding\"));\n assertEquals(\"gzip\", res.header(\"ACCEPT-ENCODING\"));\n assertEquals(\"text/html\", res.header(\"Content-Type\"));\n assertEquals(\"http://example.com\", res.header(\"Referrer\"));\n\n res.removeHeader(\"Content-Type\");\n assertFalse(res.hasHeader(\"content-type\"));\n\n res.removeHeader(\"ACCEPT-ENCODING\");\n assertFalse(res.hasHeader(\"Accept-Encoding\"));\n\n res.header(\"ACCEPT-ENCODING\", \"deflate\");\n assertEquals(\"deflate\", res.header(\"Accept-Encoding\"));\n assertEquals(\"deflate\", res.header(\"accept-Encoding\"));\n }\n\n @Test public void headers() {\n Connection con = HttpConnection.connect(\"http://example.com\");\n Map headers = new HashMap<>();\n headers.put(\"content-type\", \"text/html\");\n headers.put(\"Connection\", \"keep-alive\");\n headers.put(\"Host\", \"http://example.com\");\n con.headers(headers);\n assertEquals(\"text/html\", con.request().header(\"content-type\"));\n assertEquals(\"keep-alive\", con.request().header(\"Connection\"));\n assertEquals(\"http://example.com\", con.request().header(\"Host\"));\n }\n\n @Test public void sameHeadersCombineWithComma() {\n Map> headers = new HashMap<>();\n List values = new ArrayList<>();\n values.add(\"no-cache\");\n values.add(\"no-store\");\n headers.put(\"Cache-Control\", values);\n HttpConnection.Response res = new HttpConnection.Response();\n res.processResponseHeaders(headers);\n assertEquals(\"no-cache, no-store\", res.header(\"Cache-Control\"));\n }\n\n @Test public void multipleHeaders() {\n Connection.Request req = new HttpConnection.Request();\n req.addHeader(\"Accept\", \"Something\");\n req.addHeader(\"Accept\", \"Everything\");\n req.addHeader(\"Foo\", \"Bar\");\n\n assertTrue(req.hasHeader(\"Accept\"));\n assertTrue(req.hasHeader(\"ACCEpt\"));\n assertEquals(\"Something, Everything\", req.header(\"accept\"));\n assertTrue(req.hasHeader(\"fOO\"));\n assertEquals(\"Bar\", req.header(\"foo\"));\n\n List accept = req.headers(\"accept\");\n assertEquals(2, accept.size());\n assertEquals(\"Something\", accept.get(0));\n assertEquals(\"Everything\", accept.get(1));\n\n Map> headers = req.multiHeaders();\n assertEquals(accept, headers.get(\"Accept\"));\n assertEquals(\"Bar\", headers.get(\"Foo\").get(0));\n\n assertTrue(req.hasHeader(\"Accept\"));\n assertTrue(req.hasHeaderWithValue(\"accept\", \"Something\"));\n assertTrue(req.hasHeaderWithValue(\"accept\", \"Everything\"));\n assertFalse(req.hasHeaderWithValue(\"accept\", \"Something for nothing\"));\n\n req.removeHeader(\"accept\");\n headers = req.multiHeaders();\n assertEquals(\"Bar\", headers.get(\"Foo\").get(0));\n assertFalse(req.hasHeader(\"Accept\"));\n assertNull(headers.get(\"Accept\"));\n }\n\n @Test void responseHeadersPreserveInsertOrder() throws IOException {\n // linkedhashmap preserves the response header order\n Connection.Response res = new HttpConnection.Response();\n res.addHeader(\"5\", \"5\");\n res.addHeader(\"4\", \"4\");\n res.addHeader(\"3\", \"3\");\n res.addHeader(\"2\", \"2\");\n res.addHeader(\"1\", \"1\");\n\n String[] expected = {\"5\", \"4\", \"3\", \"2\", \"1\"};\n int i = 0;\n for (String key : res.headers().keySet()) {\n assertEquals(expected[i++], key);\n }\n\n assertInstanceOf(LinkedHashMap.class, res.headers());\n }\n\n @Test public void ignoresEmptySetCookies() {\n // prep http response header map\n Map> headers = new HashMap<>();\n headers.put(\"Set-Cookie\", Collections.emptyList());\n HttpConnection.Response res = new HttpConnection.Response();\n res.processResponseHeaders(headers);\n assertEquals(0, res.cookies().size());\n }\n\n @Test public void connectWithUrl() throws MalformedURLException {\n Connection con = HttpConnection.connect(new URL(\"http://example.com\"));\n assertEquals(\"http://example.com\", con.request().url().toExternalForm());\n }\n\n @Test public void throwsOnMalformedUrl() {\n assertThrows(IllegalArgumentException.class, () -> HttpConnection.connect(\"bzzt\"));\n }\n\n @Test public void userAgent() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n assertEquals(HttpConnection.DEFAULT_UA, con.request().header(\"User-Agent\"));\n con.userAgent(\"Mozilla\");\n assertEquals(\"Mozilla\", con.request().header(\"User-Agent\"));\n }\n\n @Test public void timeout() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n assertEquals(30 * 1000, con.request().timeout());\n con.timeout(1000);\n assertEquals(1000, con.request().timeout());\n }\n\n @Test public void referrer() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n con.referrer(\"http://foo.com\");\n assertEquals(\"http://foo.com\", con.request().header(\"Referer\"));\n }\n\n @Test public void method() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n assertEquals(Connection.Method.GET, con.request().method());\n con.method(Connection.Method.POST);\n assertEquals(Connection.Method.POST, con.request().method());\n }\n\n @Test public void throwsOnOddData() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n con.data(\"Name\", \"val\", \"what\");\n });\n }\n\n @Test public void data() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n con.data(\"Name\", \"Val\", \"Foo\", \"bar\");\n Collection values = con.request().data();\n Object[] data = values.toArray();\n Connection.KeyVal one = (Connection.KeyVal) data[0];\n Connection.KeyVal two = (Connection.KeyVal) data[1];\n assertEquals(\"Name\", one.key());\n assertEquals(\"Val\", one.value());\n assertEquals(\"Foo\", two.key());\n assertEquals(\"bar\", two.value());\n }\n\n @Test public void cookie() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n con.cookie(\"Name\", \"Val\");\n assertEquals(\"Val\", con.request().cookie(\"Name\"));\n }\n\n @Test public void inputStream() {\n Connection.KeyVal kv = HttpConnection.KeyVal.create(\"file\", \"thumb.jpg\", ParseTest.inputStreamFrom(\"Check\"));\n assertEquals(\"file\", kv.key());\n assertEquals(\"thumb.jpg\", kv.value());\n assertTrue(kv.hasInputStream());\n\n kv = HttpConnection.KeyVal.create(\"one\", \"two\");\n assertEquals(\"one\", kv.key());\n assertEquals(\"two\", kv.value());\n assertFalse(kv.hasInputStream());\n }\n\n @Test public void requestBody() {\n Connection con = HttpConnection.connect(\"http://example.com/\");\n con.requestBody(\"foo\");\n assertEquals(\"foo\", con.request().requestBody());\n }\n\n @Test public void encodeUrl() throws MalformedURLException {\n URL url1 = new URL(\"https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag\");\n URL url2 = new UrlBuilder(url1).build();\n assertEquals(\"https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag\", url2.toExternalForm());\n }\n\n @Test public void encodeUrlSupplementary() throws MalformedURLException {\n URL url1 = new URL(\"https://example.com/tools/test💩.html\"); // = \"/tools/test\\uD83D\\uDCA9.html\"\n URL url2 = new UrlBuilder(url1).build();\n assertEquals(\"https://example.com/tools/test%F0%9F%92%A9.html\", url2.toExternalForm());\n }\n\n @Test void encodedUrlDoesntDoubleEncode() throws MalformedURLException {\n URL url1 = new URL(\"https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment\");\n URL url2 = new UrlBuilder(url1).build();\n URL url3 = new UrlBuilder(url2).build();\n assertEquals(\"https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment\", url2.toExternalForm());\n assertEquals(\"https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment\", url3.toExternalForm());\n }\n\n @Test void urlPathIsPreservedDoesntDoubleEncode() throws MalformedURLException {\n URL url1 = new URL(\"https://test.com/[foo] bar+/%5BOne%5D?q=white space#frag ment\");\n URL url2 = new UrlBuilder(url1).build();\n URL url3 = new UrlBuilder(url2).build();\n assertEquals(\"https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment\", url2.toExternalForm());\n assertEquals(\"https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment\", url3.toExternalForm());\n }\n\n @Test void connectToEncodedUrl() {\n Connection connect = Jsoup.connect(\"https://example.com/a%20b%20c?query+string\");\n URL url = connect.request().url();\n assertEquals(\"https://example.com/a%20b%20c?query+string\", url.toExternalForm());\n }\n\n @Test void encodedUrlPathIsPreserved() {\n // https://github.com/jhy/jsoup/issues/1952\n Connection connect = Jsoup.connect(\"https://example.com/%2B32\");\n URL url = connect.request().url();\n assertEquals(\"https://example.com/%2B32\", url.toExternalForm());\n }\n\n @Test void urlPathPlusIsPreserved() {\n // https://github.com/jhy/jsoup/issues/1952\n Connection connect = Jsoup.connect(\"https://example.com/123+456\");\n URL url = connect.request().url();\n assertEquals(\"https://example.com/123+456\", url.toExternalForm());\n }\n\n @Test public void noUrlThrowsValidationError() throws IOException {\n HttpConnection con = new HttpConnection();\n boolean threw = false;\n try {\n con.execute();\n } catch (IllegalArgumentException e) {\n threw = true;\n assertEquals(\"URL not set. Make sure to call #url(...) before executing the request.\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test public void handlesHeaderEncodingOnRequest() {\n Connection.Request req = new HttpConnection.Request();\n req.addHeader(\"xxx\", \"é\");\n }\n\n @Test public void supportsInternationalDomainNames() throws MalformedURLException {\n String idn = \"https://www.测试.测试/foo.html?bar\";\n String puny = \"https://www.xn--0zwm56d.xn--0zwm56d/foo.html?bar\";\n\n Connection con = Jsoup.connect(idn);\n assertEquals(puny, con.request().url().toExternalForm());\n\n HttpConnection.Request req = new HttpConnection.Request();\n req.url(new URL(idn));\n assertEquals(puny, req.url().toExternalForm());\n }\n\n @Test void supportsIdnWithPort() throws MalformedURLException {\n String idn = \"https://www.测试.测试:9001/foo.html?bar\";\n String puny = \"https://www.xn--0zwm56d.xn--0zwm56d:9001/foo.html?bar\";\n\n Connection con = Jsoup.connect(idn);\n assertEquals(puny, con.request().url().toExternalForm());\n\n HttpConnection.Request req = new HttpConnection.Request();\n req.url(new URL(idn));\n assertEquals(puny, req.url().toExternalForm());\n }\n\n @Test public void validationErrorsOnExecute() throws IOException {\n Connection con = new HttpConnection();\n boolean urlThrew = false;\n try {\n con.execute();\n } catch (IllegalArgumentException e) {\n urlThrew = e.getMessage().contains(\"URL\");\n }\n assertTrue(urlThrew);\n }\n\n @Test void testMalformedException() {\n boolean threw = false;\n try {\n Jsoup.connect(\"jsoup.org/test\");\n } catch (IllegalArgumentException e) {\n threw = true;\n assertEquals(\"The supplied URL, 'jsoup.org/test', is malformed. Make sure it is an absolute URL, and starts with 'http://' or 'https://'. See https://jsoup.org/cookbook/extracting-data/working-with-urls\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test void setHeaderWithUnicodeValue() {\n Connection connect = Jsoup.connect(\"https://example.com\");\n String value = \"/foo/我的\";\n connect.header(\"Key\", value);\n\n String actual = connect.request().header(\"Key\");\n assertEquals(value, actual);\n }\n\n @Test void setAuth() throws MalformedURLException {\n Connection con = Jsoup.newSession();\n\n assertNull(con.request().auth());\n\n RequestAuthenticator auth1 = new RequestAuthenticator() {\n @Override public PasswordAuthentication authenticate(Context auth) {\n return auth.credentials(\"foo\", \"bar\");\n }\n };\n\n RequestAuthenticator auth2 = new RequestAuthenticator() {\n @Override public PasswordAuthentication authenticate(Context auth) {\n return auth.credentials(\"qux\", \"baz\");\n }\n };\n\n con.auth(auth1);\n assertSame(con.request().auth(), auth1);\n\n con.auth(auth2);\n assertSame(con.request().auth(), auth2);\n\n con.request().auth(auth1);\n assertSame(con.request().auth(), auth1);\n\n PasswordAuthentication creds = auth1.authenticate(\n new RequestAuthenticator.Context(new URL(\"http://example.com\"), Authenticator.RequestorType.SERVER, \"Realm\"));\n assertNotNull(creds);\n assertEquals(\"foo\", creds.getUserName());\n assertEquals(\"bar\", new String(creds.getPassword()));\n }\n\n /* Tests for fixHeaderEncoding. We are handling two cases when a server sends a header in UTF8. The JVM will decode\n that in 8859 (per the RFC) and we'll get mojibake, and so we try to fix it. On Android, will be decoded correctly,\n so should not be modified. */\n static String mojibake(String input) {\n // simulate mojibake by encoding in UTF-8 and decoding in ISO-8859-1\n return new String(input.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1);\n }\n\n @ParameterizedTest\n @ValueSource(strings = {\"search.php?moji=我的\", \"latin=café\", \"🍕\", \"ascii\"})\n void fixesHeaderEncodingIfRequired(String input) {\n // if the input was mojibaked, we fix it; otherwise is passed\n // https://github.com/jhy/jsoup/issues/2011\n assertEquals(input, fixHeaderEncoding(input));\n assertEquals(input, fixHeaderEncoding(mojibake(input)));\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/RegexTest.java",
"content": "package org.jsoup.helper;\n\nimport org.jsoup.select.QueryParser;\nimport org.jsoup.select.Selector;\nimport org.junit.jupiter.api.AfterEach;\nimport org.junit.jupiter.api.BeforeEach;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.ValueSource;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class RegexTest {\n\n private boolean originalUseRe2j; // track original setting\n\n @BeforeEach\n void setUp() {\n originalUseRe2j = Regex.wantsRe2j();\n }\n\n @AfterEach\n void tearDown() {\n Regex.wantsRe2j(originalUseRe2j); // restore original setting\n }\n\n @ParameterizedTest\n @ValueSource(booleans = {false, true})\n void testRegexDelegates(boolean useRe2j) {\n Regex.wantsRe2j(useRe2j);\n assertEquals(Regex.usingRe2j(), useRe2j);\n String pattern = \"(\\\\d+)\";\n String input = \"12345\";\n\n Regex regex = Regex.compile(pattern);\n Regex.Matcher matcher = regex.matcher(input);\n assertTrue(matcher.find());\n }\n\n @Test\n void jdkSupportsBackreferenceMatches() {\n Regex.wantsRe2j(false);\n String pattern = \"(\\\\w+)\\\\s+\\\\1\"; // backreference to group 1\n String input = \"hello hello\";\n\n Regex regex = Regex.compile(pattern);\n Regex.Matcher matcher = regex.matcher(input);\n assertTrue(matcher.find());\n }\n\n @Test\n void re2jRejectsBackreferenceThrows() {\n Regex.wantsRe2j(true);\n String pattern = \"(\\\\w+)\\\\s+\\\\1\"; // backreference unsupported by RE2J\n\n assertThrows(ValidationException.class, () -> Regex.compile(pattern));\n // and not the rej2 PatternSyntaxException\n }\n\n @ParameterizedTest\n @ValueSource(booleans = {false, true})\n void queryParserThrowsSelectorExceptionOnMalformedRegex(boolean useRe2j) {\n Regex.wantsRe2j(useRe2j);\n String query = \"[attr~=(unclosed]\";\n\n boolean threw = false;\n try {\n QueryParser.parse(query);\n } catch (Selector.SelectorParseException e) {\n threw = true;\n assertTrue(e.getMessage().contains(\"Pattern syntax error\"));\n }\n assertTrue(threw);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/ValidateTest.java",
"content": "package org.jsoup.helper;\n\nimport org.junit.jupiter.api.Assertions;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\n@SuppressWarnings(\"deprecation\") // keeps tests for ensureNotNull\npublic class ValidateTest {\n @Test\n public void testNotNull() {\n Validate.notNull(\"foo\");\n boolean threw = false;\n try {\n Validate.notNull(null);\n } catch (IllegalArgumentException e) {\n threw = true;\n }\n Assertions.assertTrue(threw);\n }\n\n @Test\n void stacktraceFiltersOutValidateClass() {\n boolean threw = false;\n try {\n Validate.notNull(null);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Object must not be null\", e.getMessage());\n StackTraceElement[] stackTrace = e.getStackTrace();\n for (StackTraceElement trace : stackTrace) {\n assertNotEquals(trace.getClassName(), Validate.class.getName());\n }\n assertTrue(stackTrace.length >= 1);\n }\n Assertions.assertTrue(threw);\n }\n\n @Test\n void nonnullParam() {\n boolean threw = true;\n try {\n Validate.notNullParam(null, \"foo\");\n } catch (ValidationException e) {\n assertEquals(\"The parameter 'foo' must not be null.\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testWtf() {\n boolean threw = false;\n try {\n Validate.wtf(\"Unexpected state reached\");\n } catch (IllegalStateException e) {\n threw = true;\n assertEquals(\"Unexpected state reached\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testEnsureNotNull() {\n // Test with a non-null object\n Object obj = new Object();\n assertSame(obj, Validate.ensureNotNull(obj));\n\n // Test with a null object\n boolean threw = false;\n try {\n Validate.ensureNotNull(null);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Object must not be null\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testEnsureNotNullWithMessage() {\n // Test with a non-null object\n Object obj = new Object();\n assertSame(obj, Validate.ensureNotNull(obj, \"Object must not be null\"));\n\n // Test with a null object\n boolean threw = false;\n try {\n Validate.ensureNotNull(null, \"Custom error message\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Custom error message\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testEnsureNotNullWithFormattedMessage() {\n // Test with a non-null object\n Object obj = new Object();\n assertSame(obj, Validate.ensureNotNull(obj, \"Object must not be null: %s\", \"additional info\"));\n\n // Test with a null object\n boolean threw = false;\n try {\n Validate.ensureNotNull(null, \"Object must not be null: %s\", \"additional info\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Object must not be null: additional info\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test void expectNotNull() {\n String foo = \"Foo\";\n String foo2 = Validate.expectNotNull(foo);\n assertSame(foo, foo2);\n\n // Test with a null object\n String bar = null;\n boolean threw = false;\n try {\n Validate.expectNotNull(bar);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Object must not be null\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testNotNullParam() {\n // Test with a non-null object\n Object obj = new Object();\n Validate.notNullParam(obj, \"param\");\n\n // Test with a null object\n boolean threw = false;\n try {\n Validate.notNullParam(null, \"param\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"The parameter 'param' must not be null.\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testNotEmpty() {\n // Test with a non-empty string\n String str = \"foo\";\n Validate.notEmpty(str);\n\n // Test with an empty string\n boolean threw = false;\n try {\n Validate.notEmpty(\"\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"String must not be empty\", e.getMessage());\n }\n assertTrue(threw);\n\n // Test with a null string\n threw = false;\n try {\n Validate.notEmpty(null);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"String must not be empty\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testIsTrue() {\n // Test with a true value\n Validate.isTrue(true);\n\n // Test with a false value\n boolean threw = false;\n try {\n Validate.isTrue(false);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Must be true\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testIsFalse() {\n // Test with a false value\n Validate.isFalse(false);\n\n // Test with a true value\n boolean threw = false;\n try {\n Validate.isFalse(true);\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Must be false\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testAssertFail() {\n boolean result = false;\n boolean threw = false;\n try {\n result = Validate.assertFail(\"This should fail\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"This should fail\", e.getMessage());\n }\n assertTrue(threw);\n assertFalse(result);\n }\n\n @Test\n public void testNotEmptyParam() {\n // Test with a non-empty string\n Validate.notEmptyParam(\"foo\", \"param\");\n\n // Test with an empty string\n boolean threw = false;\n try {\n Validate.notEmptyParam(\"\", \"param\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"The 'param' parameter must not be empty.\", e.getMessage());\n }\n assertTrue(threw);\n\n // Test with a null string\n threw = false;\n try {\n Validate.notEmptyParam(null, \"param\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"The 'param' parameter must not be empty.\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testNoNullElementsWithMessage() {\n // Test with an array with no null elements\n Object[] array = {new Object(), new Object()};\n Validate.noNullElements(array, \"Custom error message\");\n\n // Test with an array containing a null element\n boolean threw = false;\n try {\n Validate.noNullElements(new Object[]{new Object(), null}, \"Custom error message\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Custom error message\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test\n public void testNotEmptyWithMessage() {\n // Test with a non-empty string\n Validate.notEmpty(\"foo\", \"Custom error message\");\n\n // Test with an empty string\n boolean threw = false;\n try {\n Validate.notEmpty(\"\", \"Custom error message\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Custom error message\", e.getMessage());\n }\n assertTrue(threw);\n\n // Test with a null string\n threw = false;\n try {\n Validate.notEmpty(null, \"Custom error message\");\n } catch (ValidationException e) {\n threw = true;\n assertEquals(\"Custom error message\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/helper/W3CDomTest.java",
"content": "package org.jsoup.helper;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.TextUtil;\nimport org.jsoup.integration.ParseTest;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.nodes.TextNode;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.Arguments;\nimport org.junit.jupiter.params.provider.MethodSource;\nimport org.w3c.dom.Document;\nimport org.w3c.dom.Node;\nimport org.w3c.dom.NodeList;\nimport org.xml.sax.InputSource;\n\nimport javax.xml.parsers.DocumentBuilder;\nimport javax.xml.parsers.DocumentBuilderFactory;\nimport javax.xml.parsers.ParserConfigurationException;\nimport javax.xml.transform.OutputKeys;\nimport javax.xml.xpath.XPathConstants;\nimport javax.xml.xpath.XPathExpression;\nimport javax.xml.xpath.XPathExpressionException;\nimport javax.xml.xpath.XPathFactory;\nimport java.io.ByteArrayInputStream;\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.StringReader;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.stream.Stream;\n\nimport static org.jsoup.TextUtil.normalizeSpaces;\nimport static org.jsoup.nodes.Document.OutputSettings.Syntax.xml;\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class W3CDomTest {\n\n private static Document parseXml(String xml, boolean nameSpaceAware) {\n try {\n DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();\n factory.setNamespaceAware(nameSpaceAware);\n DocumentBuilder builder = factory.newDocumentBuilder();\n builder.setEntityResolver((publicId, systemId) -> {\n if (systemId.contains(\"about:legacy-compat\")) { // \n return new InputSource(new StringReader(\"\"));\n } else {\n return null;\n }\n });\n Document dom = builder.parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));\n dom.normalizeDocument();\n return dom;\n } catch (Exception e) {\n throw new IllegalStateException(e);\n }\n }\n\n @Test\n public void simpleConversion() {\n String html = \"W3cText
What\";\n assertEquals(expected, TextUtil.stripNewlines(out));\n\n Document roundTrip = parseXml(out, true);\n assertEquals(\"Text\", roundTrip.getElementsByTagName(\"p\").item(0).getTextContent());\n\n // check we can set properties\n Map properties = W3CDom.OutputXml();\n properties.put(OutputKeys.INDENT, \"yes\");\n String furtherOut = W3CDom.asString(wDoc, properties);\n assertTrue(furtherOut.length() > out.length()); // wanted to assert formatting, but actual indentation is platform specific so breaks in CI\n String furtherExpected =\n \"W3cText
What\";\n assertEquals(furtherExpected, TextUtil.stripNewlines(furtherOut)); // on windows, DOM will write newlines as \\r\\n\n }\n\n @Test\n public void namespacePreservation() throws IOException {\n File in = ParseTest.getFile(\"/htmltests/namespaces.xhtml\");\n org.jsoup.nodes.Document jsoupDoc;\n jsoupDoc = Jsoup.parse(in, \"UTF-8\", \"\", Parser.xmlParser());\n\n Document doc;\n org.jsoup.helper.W3CDom jDom = new org.jsoup.helper.W3CDom();\n doc = jDom.fromJsoup(jsoupDoc);\n\n Node htmlEl = doc.getChildNodes().item(0);\n assertEquals(\"http://www.w3.org/1999/xhtml\", htmlEl.getNamespaceURI());\n assertEquals(\"html\", htmlEl.getLocalName());\n assertEquals(\"html\", htmlEl.getNodeName());\n\n // inherits default namespace\n Node head = htmlEl.getFirstChild().getNextSibling();\n assertEquals(\"http://www.w3.org/1999/xhtml\", head.getNamespaceURI());\n assertEquals(\"head\", head.getLocalName());\n assertEquals(\"head\", head.getNodeName());\n\n Node epubTitle = htmlEl.getChildNodes().item(3).getChildNodes().item(3);\n assertEquals(\"Check\", epubTitle.getTextContent());\n assertEquals(\"http://www.idpf.org/2007/ops\", epubTitle.getNamespaceURI());\n assertEquals(\"title\", epubTitle.getLocalName());\n assertEquals(\"epub:title\", epubTitle.getNodeName());\n\n Node xSection = epubTitle.getNextSibling().getNextSibling();\n assertEquals(\"urn:test\", xSection.getNamespaceURI());\n assertEquals(\"section\", xSection.getLocalName());\n assertEquals(\"x:section\", xSection.getNodeName());\n\n // https://github.com/jhy/jsoup/issues/977\n // does not keep last set namespace\n Node svg = xSection.getNextSibling().getNextSibling();\n assertEquals(\"http://www.w3.org/2000/svg\", svg.getNamespaceURI());\n assertEquals(\"svg\", svg.getLocalName());\n assertEquals(\"svg\", svg.getNodeName());\n\n Node path = svg.getChildNodes().item(1);\n assertEquals(\"http://www.w3.org/2000/svg\", path.getNamespaceURI());\n assertEquals(\"path\", path.getLocalName());\n assertEquals(\"path\", path.getNodeName());\n\n Node clip = path.getChildNodes().item(1);\n assertEquals(\"http://example.com/clip\", clip.getNamespaceURI());\n assertEquals(\"clip\", clip.getLocalName());\n assertEquals(\"clip\", clip.getNodeName());\n assertEquals(\"456\", clip.getTextContent());\n\n Node picture = svg.getNextSibling().getNextSibling();\n assertEquals(\"http://www.w3.org/1999/xhtml\", picture.getNamespaceURI());\n assertEquals(\"picture\", picture.getLocalName());\n assertEquals(\"picture\", picture.getNodeName());\n\n Node img = picture.getFirstChild();\n assertEquals(\"http://www.w3.org/1999/xhtml\", img.getNamespaceURI());\n assertEquals(\"img\", img.getLocalName());\n assertEquals(\"img\", img.getNodeName());\n }\n\n @Test\n public void handlesInvalidAttributeNames() {\n String html = \"\";\n org.jsoup.nodes.Document jsoupDoc;\n jsoupDoc = Jsoup.parse(html);\n Element body = jsoupDoc.select(\"body\").first();\n assertTrue(body.hasAttr(\"\\\"\")); // actually an attribute with key '\"'. Correct per HTML5 spec, but w3c xml dom doesn't dig it\n assertTrue(body.hasAttr(\"name\\\"\"));\n\n Document w3Doc = W3CDom.convert(jsoupDoc);\n String xml = W3CDom.asString(w3Doc, W3CDom.OutputXml());\n assertEquals(\"\", xml);\n }\n\n @Test\n public void htmlInputDocMaintainsHtmlAttributeNames() {\n String html = \"unicode attr names
\";\n org.jsoup.nodes.Document jsoupDoc;\n jsoupDoc = Jsoup.parse(html);\n\n Document w3Doc = W3CDom.convert(jsoupDoc);\n String out = W3CDom.asString(w3Doc, W3CDom.OutputHtml());\n String expected = \"unicode attr names
\";\n assertEquals(expected, TextUtil.stripNewlines(out));\n }\n\n @Test\n public void xmlInputDocMaintainsHtmlAttributeNames() {\n String html = \"unicode attr names coerced
\";\n org.jsoup.nodes.Document jsoupDoc;\n jsoupDoc = Jsoup.parse(html);\n jsoupDoc.outputSettings().syntax(xml);\n\n Document w3Doc = W3CDom.convert(jsoupDoc);\n String out = W3CDom.asString(w3Doc, W3CDom.OutputHtml());\n String expected = \"unicode attr names coerced
\";\n assertEquals(expected, TextUtil.stripNewlines(out));\n }\n\n @Test\n public void handlesInvalidTagAsText() {\n org.jsoup.nodes.Document jsoup = Jsoup.parse(\"<インセンティブで高収入！>Text More
\");\n\n Document w3Doc = W3CDom.convert(jsoup);\n String xml = W3CDom.asString(w3Doc, W3CDom.OutputXml());\n assertEquals(\"<インセンティブで高収入！>Text More
\", xml);\n }\n\n @Test void handlesHtmlElsWithLt() {\n // In HTML, elements can be named \"foo). Test that we can convert to W3C, that we can HTML parse our HTML serial, XML parse our XML serial, and W3C XML parse the XML serial and the W3C serial\n // And similarly attributes may have \"<\" in their name\n // https://github.com/jhy/jsoup/issues/2259\n String input = \"Text\";\n String xmlExpect = \"Text\"; // rewrites < to _ in el and attr\n\n // html round trips\n org.jsoup.nodes.Document htmlDoc = Jsoup.parse(input);\n String htmlSerial = htmlDoc.body().html();\n assertEquals(input, normalizeSpaces(htmlSerial)); // same as input\n Element htmlRound = Jsoup.parse(htmlSerial).body();\n assertTrue(htmlDoc.body().hasSameValue(htmlRound));\n\n // xml round trips\n htmlDoc.outputSettings().syntax(xml);\n String asXml = htmlDoc.body().html();\n assertEquals(xmlExpect, normalizeSpaces(asXml)); // -> \n org.jsoup.nodes.Document xmlDoc = Jsoup.parse(asXml);\n String xmlSerial = xmlDoc.body().html();\n assertEquals(xmlExpect, normalizeSpaces(xmlSerial)); // same as xmlExpect\n Element xmlRound = Jsoup.parse(xmlSerial).body();\n assertTrue(xmlDoc.body().hasSameValue(xmlRound));\n\n // Can W3C parse that XML\n Document w3cXml = parseXml(asXml, true);\n NodeList w3cXmlNodes = w3cXml.getElementsByTagName(\"foo_bar\");\n assertEquals(1, w3cXmlNodes.getLength());\n assertEquals(\"123\", w3cXmlNodes.item(0).getAttributes().getNamedItem(\"attr_name\").getTextContent());\n\n // Can convert to W3C\n Document w3cDoc = W3CDom.convert(htmlDoc);\n NodeList w3cNodes = w3cDoc.getElementsByTagName(\"foo_bar\");\n assertEquals(1, w3cNodes.getLength());\n assertEquals(\"123\", w3cNodes.item(0).getAttributes().getNamedItem(\"attr_name\").getTextContent());\n }\n\n @Test\n public void canConvertToCustomDocument() throws ParserConfigurationException {\n org.jsoup.nodes.Document document = Jsoup.parse(\"\");\n\n DocumentBuilderFactory localDocumentBuilderFactory = DocumentBuilderFactory.newInstance();\n Document customDocumentResult = localDocumentBuilderFactory.newDocumentBuilder().newDocument();\n\n W3CDom w3cDom = new W3CDom();\n w3cDom.convert(document, customDocumentResult);\n\n String html = W3CDom.asString(customDocumentResult, W3CDom.OutputHtml());\n assertEquals(\"\", html);\n }\n\n @Test\n public void treatsUndeclaredNamespaceAsLocalName() {\n String html = \"One\";\n org.jsoup.nodes.Document doc = Jsoup.parse(html);\n\n Document w3Doc = new W3CDom().fromJsoup(doc);\n Node htmlEl = w3Doc.getFirstChild();\n\n assertEquals(\"http://www.w3.org/1999/xhtml\", htmlEl.getNamespaceURI());\n assertEquals(\"html\", htmlEl.getLocalName());\n assertEquals(\"html\", htmlEl.getNodeName());\n\n Node fb = htmlEl.getFirstChild().getNextSibling().getFirstChild();\n assertEquals(\"http://www.w3.org/1999/xhtml\", fb.getNamespaceURI());\n assertEquals(\"like\", fb.getLocalName());\n assertEquals(\"fb:like\", fb.getNodeName());\n }\n\n @Test\n public void xmlnsXpathTest() throws XPathExpressionException {\n W3CDom w3c = new W3CDom();\n String html = \"hello
\";\n Document dom = w3c.fromJsoup(Jsoup.parse(html));\n NodeList nodeList = xpath(dom, \"//*[local-name()=\\\"body\\\"]\");// namespace aware; HTML namespace is default\n assertEquals(\"div\", nodeList.item(0).getLocalName());\n\n // default output is namespace aware, so query needs to be as well\n html = \"hello
\";\n dom = w3c.fromJsoup(Jsoup.parse(html));\n nodeList = xpath(dom, \"//body\");\n assertNull(nodeList); // no matches\n\n dom = w3c.fromJsoup(Jsoup.parse(html));\n nodeList = xpath(dom, \"//*[local-name()=\\\"body\\\"]\");\n assertNotNull(nodeList);\n assertEquals(1, nodeList.getLength());\n assertEquals(\"div\", nodeList.item(0).getLocalName());\n assertEquals(\"http://www.w3.org/1999/xhtml\", nodeList.item(0).getNamespaceURI());\n assertNull(nodeList.item(0).getPrefix());\n\n // get rid of the name space awareness\n String xml = w3c.asString(dom);\n dom = parseXml(xml, false);\n Node item = (Node) xpath(dom, \"//body\");\n assertEquals(\"body\", item.getNodeName());\n assertNull(item.getNamespaceURI());\n assertNull(item.getPrefix());\n\n // put back, will get zero\n dom = parseXml(xml, true);\n nodeList = xpath(dom, \"//body\");\n assertNull(nodeList);\n }\n\n @Test\n public void xhtmlNoNamespace() throws XPathExpressionException {\n W3CDom w3c = new W3CDom();\n String html = \"hello
\";\n w3c.namespaceAware(false);\n Document dom = w3c.fromJsoup(Jsoup.parse(html));\n NodeList nodeList = xpath(dom, \"//body\");// no namespace\n assertEquals(1, nodeList.getLength());\n assertEquals(\"div\", nodeList.item(0).getLocalName());\n }\n\n @Test\n void canDisableNamespaces() throws XPathExpressionException {\n W3CDom w3c = new W3CDom();\n assertTrue(w3c.namespaceAware());\n\n w3c.namespaceAware(false);\n assertFalse(w3c.namespaceAware());\n\n String html = \"hello
\";\n Document dom = w3c.fromJsoup(Jsoup.parse(html));\n NodeList nodeList = xpath(dom, \"//body\");// no ns, so needs no prefix\n assertEquals(\"div\", nodeList.item(0).getLocalName());\n }\n\n private NodeList xpath(Document w3cDoc, String query) throws XPathExpressionException {\n XPathExpression xpath = XPathFactory.newInstance().newXPath().compile(query);\n return ((NodeList) xpath.evaluate(w3cDoc, XPathConstants.NODE));\n }\n\n @Test\n public void testRoundTripDoctype() {\n // because we have Saxon on the test classpath, the transformer will change to that, and so case may change (e.g. Java base is META, Saxon is meta for HTML)\n String base = \"One
\";\n assertEqualsIgnoreCase(\"One
\", output(base, true));\n assertEqualsIgnoreCase(\"One
\", output(base, false));\n\n String publicDoc = \"\";\n assertEqualsIgnoreCase(\"\", output(publicDoc, true));\n // different impls will have different XML formatting. OpenJDK 13 default gives this: but others have , so just check start\n assertTrue(output(publicDoc, false).startsWith(\"\";\n assertEqualsIgnoreCase(\"\", output(systemDoc, true));\n assertEqualsIgnoreCase(\"\", output(systemDoc, false));\n\n String legacyDoc = \"\";\n assertEqualsIgnoreCase(\"\", output(legacyDoc, true));\n assertEqualsIgnoreCase(\"\", output(legacyDoc, false));\n\n String noDoctype = \"One
\";\n assertEqualsIgnoreCase(\"One
\", output(noDoctype, true));\n assertEqualsIgnoreCase(\"One
\", output(noDoctype, false));\n }\n\n private String output(String in, boolean modeHtml) {\n org.jsoup.nodes.Document jdoc = Jsoup.parse(in);\n Document w3c = W3CDom.convert(jdoc);\n\n Map properties = modeHtml ? W3CDom.OutputHtml() : W3CDom.OutputXml();\n return normalizeSpaces(W3CDom.asString(w3c, properties));\n }\n\n private void assertEqualsIgnoreCase(String want, String have) {\n assertEquals(want.toLowerCase(Locale.ROOT), have.toLowerCase(Locale.ROOT));\n }\n\n\n @Test\n public void canOutputHtmlWithoutNamespace() {\n String html = \"One
\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n W3CDom w3c = new W3CDom();\n w3c.namespaceAware(false);\n\n String asHtml = W3CDom.asString(w3c.fromJsoup(jdoc), W3CDom.OutputHtml());\n String asXtml = W3CDom.asString(w3c.fromJsoup(jdoc), W3CDom.OutputXml());\n assertEqualsIgnoreCase(\n \"one
\",\n asHtml);\n assertEqualsIgnoreCase(\n \"One
\",\n asXtml);\n }\n\n @Test public void convertsElementsAndMaintainsSource() {\n org.jsoup.nodes.Document jdoc = Jsoup.parse(\"Two\");\n W3CDom w3CDom = new W3CDom();\n Element jDiv = jdoc.selectFirst(\"div\");\n assertNotNull(jDiv);\n Document doc = w3CDom.fromJsoup(jDiv);\n Node div = w3CDom.contextNode(doc);\n\n assertEquals(\"div\", div.getLocalName());\n assertEquals(jDiv, div.getUserData(W3CDom.SourceProperty));\n\n Node textNode = div.getFirstChild().getFirstChild();\n assertEquals(\"One\", textNode.getTextContent());\n assertEquals(Node.TEXT_NODE, textNode.getNodeType());\n\n org.jsoup.nodes.TextNode jText = (TextNode) jDiv.childNode(0).childNode(0);\n assertEquals(jText, textNode.getUserData(W3CDom.SourceProperty));\n }\n \n @Test public void canXmlParseCdataNodes() throws XPathExpressionException {\n String html = \"
5 && 6
\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n jdoc.outputSettings().syntax(xml);\n String xml = jdoc.body().html();\n assertTrue(xml.contains(\"\")); // as asserted in ElementTest\n Document doc = parseXml(xml, false);\n NodeList list = xpath(doc, \"//script\");\n assertEquals(2, list.getLength());\n Node scriptComment = list.item(0); // will be the cdata node\n assertEquals(\"//\", scriptComment.getTextContent());\n Node script = list.item(1);\n assertEquals(\"\\n\" +\n \"1 && 2\\n\" +\n \"//\", script.getTextContent());\n\n }\n\n @Test public void handlesEmptyDoctype() {\n String html = \"Foo\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n Document doc = (new W3CDom()).fromJsoup(jdoc);\n assertNull(doc.getDoctype());\n assertEquals(\"Foo\", doc.getFirstChild().getTextContent());\n }\n\n @Test void testHtmlParseAttributesAreCaseInsensitive() throws IOException {\n // https://github.com/jhy/jsoup/issues/981\n String html = \"\\n\" +\n \"\\n\" +\n \"
![\\\"Alt](\\\"firstImage.jpg\\\")
\\n\" +\n \"
![\\\"Alt](\\\"secondImage.jpg\\\")
\\n\" +\n \"\\n\" +\n \"\";\n org.jsoup.nodes.Document jsoupDoc;\n jsoupDoc = Jsoup.parse(html);\n org.jsoup.helper.W3CDom jDom = new org.jsoup.helper.W3CDom();\n Document doc = jDom.fromJsoup(jsoupDoc);\n org.w3c.dom.Element body = (org.w3c.dom.Element) doc.getDocumentElement().getElementsByTagName(\"body\").item(0);\n NodeList imgs = body.getElementsByTagName(\"img\");\n assertEquals(2, imgs.getLength());\n org.w3c.dom.Element first = (org.w3c.dom.Element) imgs.item(0);\n assertEquals(first.getAttributes().getLength(), 2);\n String img1 = first.getAttribute(\"src\");\n assertEquals(\"firstImage.jpg\", img1);\n String alt1 = first.getAttribute(\"alt\");\n assertEquals(\"Alt one\", alt1);\n org.w3c.dom.Element second = (org.w3c.dom.Element) imgs.item(1);\n assertEquals(second.getAttributes().getLength(), 2);\n String img2 = second.getAttribute(\"src\");\n assertEquals(\"secondImage.jpg\", img2);\n String alt2 = second.getAttribute(\"alt\");\n assertEquals(\"Alt two\", alt2);\n }\n\n @ParameterizedTest\n @MethodSource(\"parserProvider\")\n void doesNotExpandEntities(Parser parser) {\n // Tests that the billion laughs attack doesn't expand entities; also for XXE\n // Not impacted because jsoup doesn't parse the entities within the doctype, and so won't get to the w3c.\n // Added to confirm, and catch if that ever changes\n String billionLaughs = \"\\n\" +\n \"\\n\" +\n \" \\n\" +\n \"]>\\n\" +\n \"
&lol1;
\";\n\n org.jsoup.nodes.Document jsoupDoc = Jsoup.parse(billionLaughs, parser);\n W3CDom w3cDom = new W3CDom();\n\n org.w3c.dom.Document w3cDoc = w3cDom.fromJsoup(jsoupDoc);\n assertNotNull(w3cDoc);\n // select the p and make sure it's unexpanded\n NodeList p = w3cDoc.getElementsByTagName(\"p\");\n assertEquals(1, p.getLength());\n assertEquals(\"&lol1;\", p.item(0).getTextContent());\n\n // Check the string\n String string = W3CDom.asString(w3cDoc, W3CDom.OutputXml());\n assertFalse(string.contains(\"lololol\"));\n assertTrue(string.contains(\"&lol1;\"));\n }\n\n @Test void undeclaredAttrNamespaceAsString() {\n // https://github.com/jhy/jsoup/issues/2087\n W3CDom w3CDom = new W3CDom();\n String html = \"
\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);\n\n String xml = w3CDom.asString(w3CDoc);\n assertEquals(\"
\", xml);\n }\n\n @Test void declaredNamespaceIsUsed() {\n W3CDom w3CDom = new W3CDom();\n String html = \"
\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);\n\n String xml = w3CDom.asString(w3CDoc);\n assertEquals(\"
\", xml);\n }\n\n @Test void nestedElementsWithUndeclaredNamespace() {\n W3CDom w3CDom = new W3CDom();\n String html = \"
\";\n org.jsoup.nodes.Document jdoc = Jsoup.parse(html);\n org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);\n\n String xml = w3CDom.asString(w3CDoc);\n assertEquals(\"
\", xml);\n }\n\n private static Stream parserProvider() {\n return Stream.of(\n Arguments.of(Parser.htmlParser()),\n Arguments.of(Parser.xmlParser())\n );\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/Benchmark.java",
"content": "package org.jsoup.integration;\n\nimport java.util.Date;\n\n/**\n Does an A/B test on two methods, and prints out how long each took.\n\n @author Jonathan Hedley, jonathan@hedley.net */\npublic class Benchmark {\n public static void run(Runnable a, Runnable b, int count) {\n long aMillis;\n long bMillis;\n\n print(\"Running test A (x%d)\", count);\n aMillis = time(a, count);\n print(\"Running test B\");\n bMillis = time(b, count);\n\n print(\"\\nResults:\");\n print(\"A: %.2fs\", aMillis / 1000f);\n print(\"B: %.2fs\", bMillis / 1000f);\n print(\"\\nB ran in %.2f %% time of A\\n\", (bMillis *1f / aMillis * 1f) * 100f);\n }\n\n private static long time(Runnable test, int count) {\n Date start = new Date();\n for (int i = 0; i < count; i++) {\n test.run();\n }\n Date end = new Date();\n return end.getTime() - start.getTime();\n }\n\n private static void print(String msgFormat, Object... msgParams) {\n System.out.println(String.format(msgFormat, msgParams));\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/ConnectIT.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Connection;\nimport org.jsoup.Jsoup;\nimport org.jsoup.helper.DataUtil;\nimport org.jsoup.integration.servlets.EchoServlet;\nimport org.jsoup.integration.servlets.FileServlet;\nimport org.jsoup.integration.servlets.SlowRider;\nimport org.jsoup.internal.SharedConstants;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.parser.StreamParser;\nimport org.junit.jupiter.api.BeforeAll;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.BufferedInputStream;\nimport java.io.IOException;\nimport java.io.UncheckedIOException;\nimport java.net.SocketTimeoutException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\n/**\n * Failsafe integration tests for Connect methods. These take a bit longer to run, so included as Integ, not Unit, tests.\n */\npublic class ConnectIT {\n @BeforeAll\n public static void setUp() {\n TestServer.start();\n System.setProperty(SharedConstants.UseHttpClient, \"false\"); // use the default UrlConnection. See HttpClientConnectIT for other version\n }\n\n // Slow Rider tests.\n @Test\n public void canInterruptBodyStringRead() throws InterruptedException {\n final String[] body = new String[1];\n Thread runner = new Thread(() -> {\n try {\n Connection.Response res = Jsoup.connect(SlowRider.Url)\n .timeout(15 * 1000)\n .execute();\n body[0] = res.body();\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n\n });\n\n runner.start();\n Thread.sleep(1000 * 3);\n runner.interrupt();\n assertTrue(runner.isInterrupted());\n runner.join();\n\n assertTrue(body[0].length() > 0);\n assertTrue(body[0].contains(\"Are you still there?\"));\n }\n\n @Test\n public void canInterruptDocumentRead() throws InterruptedException {\n long start = System.currentTimeMillis();\n final String[] body = new String[1];\n Thread runner = new Thread(() -> {\n try {\n Connection.Response res = Jsoup.connect(SlowRider.Url)\n .timeout(15 * 1000)\n .execute();\n body[0] = res.parse().text();\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n\n });\n\n runner.start();\n Thread.sleep(3 * 1000);\n runner.interrupt();\n assertTrue(runner.isInterrupted());\n runner.join();\n\n long end = System.currentTimeMillis();\n // check we are between 3 and connect timeout seconds (should be just over 3; but allow some slack for slow CI runners)\n assertTrue(end - start > 3 * 1000);\n assertTrue(end - start < 10 * 1000);\n }\n\n @Test public void canInterruptThenJoinASpawnedThread() throws InterruptedException {\n // https://github.com/jhy/jsoup/issues/1991\n AtomicBoolean ioException = new AtomicBoolean();\n Thread runner = new Thread(() -> {\n try {\n while (!Thread.currentThread().isInterrupted()) {\n Document doc = Jsoup.connect(SlowRider.Url)\n .timeout(30000)\n .get();\n }\n } catch (IOException e) {\n ioException.set(true); // don't expect to catch, because the outer sleep will complete before this timeout\n }\n });\n\n runner.start();\n Thread.sleep(2 * 1000);\n runner.interrupt();\n runner.join();\n assertFalse(ioException.get());\n }\n\n @Test\n public void totalTimeout() throws IOException {\n int timeout = 3 * 1000;\n long start = System.currentTimeMillis();\n boolean threw = false;\n try {\n Jsoup.connect(SlowRider.Url).timeout(timeout).get();\n } catch (SocketTimeoutException e) {\n long end = System.currentTimeMillis();\n long took = end - start;\n assertTrue(took > timeout, (\"Time taken was \" + took));\n assertTrue(took < timeout * 1.8, (\"Time taken was \" + took));\n threw = true;\n }\n\n assertTrue(threw);\n }\n\n @Test\n public void slowReadOk() throws IOException {\n // make sure that a slow read that is under the request timeout is still OK\n Document doc = Jsoup.connect(SlowRider.Url)\n .data(SlowRider.MaxTimeParam, \"2000\") // the request completes in 2 seconds\n .get();\n\n Element h1 = doc.selectFirst(\"h1\");\n assertEquals(\"outatime\", h1.text());\n }\n\n @Test void readFullyThrowsOnTimeout() throws IOException {\n // tests that response.readFully excepts on timeout\n boolean caught = false;\n Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();\n try {\n res.readFully();\n } catch (IOException e) {\n caught = true;\n }\n assertTrue(caught);\n }\n\n @Test void readBodyThrowsOnTimeout() throws IOException {\n // tests that response.readBody excepts on timeout\n boolean caught = false;\n Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();\n try {\n res.readBody();\n } catch (IOException e) {\n caught = true;\n }\n assertTrue(caught);\n }\n\n @Test void bodyThrowsUncheckedOnTimeout() throws IOException {\n // tests that response.body unchecked excepts on timeout\n boolean caught = false;\n Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();\n try {\n res.body();\n } catch (UncheckedIOException e) {\n caught = true;\n }\n assertTrue(caught);\n }\n\n @Test\n public void infiniteReadSupported() throws IOException {\n Document doc = Jsoup.connect(SlowRider.Url)\n .timeout(0)\n .data(SlowRider.MaxTimeParam, \"2000\")\n .get();\n\n Element h1 = doc.selectFirst(\"h1\");\n assertEquals(\"outatime\", h1.text());\n }\n\n @Test void streamParserUncheckedExceptionOnTimeoutInStream() throws IOException {\n boolean caught = false;\n try (StreamParser streamParser = Jsoup.connect(SlowRider.Url)\n .data(SlowRider.MaxTimeParam, \"10000\")\n .data(SlowRider.IntroSizeParam, \"8000\") // 8K to pass first buffer, or the timeout would occur in execute or streamparser()\n .timeout(4000) // has a 1000 sleep at the start\n .execute()\n .streamParser()) {\n\n // we should expect to timeout while in stream\n try {\n long count = streamParser.stream().count();\n } catch (Exception e) {\n caught = true;\n UncheckedIOException ioe = (UncheckedIOException) e;\n IOException cause = ioe.getCause();\n //assertInstanceOf(SocketTimeoutException.class, cause); // different JDKs seem to wrap this differently\n assertInstanceOf(IOException.class, cause);\n\n }\n }\n assertTrue(caught);\n }\n\n @Test void streamParserCheckedExceptionOnTimeoutInSelect() throws IOException {\n boolean caught = false;\n try (StreamParser streamParser = Jsoup.connect(SlowRider.Url)\n .data(SlowRider.MaxTimeParam, \"10000\")\n .data(SlowRider.IntroSizeParam, \"8000\") // 8K to pass first buffer, or the timeout would occur in execute or streamparser()\n .timeout(4000) // has a 1000 sleep at the start\n .execute()\n .streamParser()) {\n\n // we should expect to timeout while in stream\n try {\n long count = 0;\n while (streamParser.selectNext(\"p\") != null) {\n count++;\n }\n } catch (IOException e) {\n caught = true;\n }\n }\n assertTrue(caught);\n }\n\n private static final int LargeHtmlSize = 280735;\n\n @Test\n public void remainingAfterFirstRead() throws IOException {\n int bufferSize = 5 * 1024;\n int capSize = 100 * 1024;\n\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n\n try (BufferedInputStream stream = Jsoup.connect(url).maxBodySize(capSize)\n .execute().bodyStream()) {\n\n // simulates parse which does a limited read first\n stream.mark(bufferSize);\n ByteBuffer firstBytes = DataUtil.readToByteBuffer(stream, bufferSize);\n\n byte[] array = firstBytes.array();\n String firstText = new String(array, StandardCharsets.UTF_8);\n assertTrue(firstText.startsWith(\"
Large\"));\n assertEquals(bufferSize, array.length);\n\n boolean fullyRead = stream.read() == -1;\n assertFalse(fullyRead);\n\n // reset and read again\n stream.reset();\n ByteBuffer fullRead = DataUtil.readToByteBuffer(stream, 0);\n byte[] fullArray = fullRead.array();\n\n // bodyStream is not capped to body size - only for jsoup consumed stream\n assertTrue(fullArray.length > capSize);\n\n assertEquals(LargeHtmlSize, fullRead.limit());\n String fullText = new String(fullRead.array(), 0, fullRead.limit(), StandardCharsets.UTF_8);\n assertTrue(fullText.startsWith(firstText));\n assertEquals(LargeHtmlSize, fullText.length());\n }\n }\n\n @Test\n public void noLimitAfterFirstRead() throws IOException {\n int firstMaxRead = 5 * 1024;\n\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n try (BufferedInputStream stream = Jsoup.connect(url).execute().bodyStream()) {\n // simulates parse which does a limited read first\n stream.mark(firstMaxRead);\n ByteBuffer firstBytes = DataUtil.readToByteBuffer(stream, firstMaxRead);\n byte[] array = firstBytes.array();\n String firstText = new String(array, StandardCharsets.UTF_8);\n assertTrue(firstText.startsWith(\"<html><head><title>Large\"));\n assertEquals(firstMaxRead, array.length);\n\n // reset and read fully\n stream.reset();\n ByteBuffer fullRead = DataUtil.readToByteBuffer(stream, 0);\n assertEquals(LargeHtmlSize, fullRead.limit());\n String fullText = new String(fullRead.array(), 0, fullRead.limit(), StandardCharsets.UTF_8);\n assertTrue(fullText.startsWith(firstText));\n assertEquals(LargeHtmlSize, fullText.length());\n }\n }\n\n @Test public void bodyStreamConstrainedViaReadFully() throws IOException {\n int cap = 5 * 1024;\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n try (BufferedInputStream stream = Jsoup\n .connect(url)\n .maxBodySize(cap)\n .execute()\n .readFully()\n .bodyStream()) {\n\n ByteBuffer cappedRead = DataUtil.readToByteBuffer(stream, 0);\n assertEquals(cap, cappedRead.limit());\n }\n }\n\n @Test public void bodyStreamConstrainedViaBufferUp() throws IOException {\n int cap = 5 * 1024;\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n try (BufferedInputStream stream = Jsoup\n .connect(url)\n .maxBodySize(cap)\n .execute()\n .bufferUp()\n .bodyStream()) {\n\n ByteBuffer cappedRead = DataUtil.readToByteBuffer(stream, 0);\n assertEquals(cap, cappedRead.limit());\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/ConnectTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Connection;\nimport org.jsoup.HttpStatusException;\nimport org.jsoup.Jsoup;\nimport org.jsoup.Connection.Method;\nimport org.jsoup.TextUtil;\nimport org.jsoup.UnsupportedMimeTypeException;\nimport org.jsoup.helper.DataUtil;\nimport org.jsoup.helper.W3CDom;\nimport org.jsoup.integration.servlets.*;\nimport org.jsoup.internal.SharedConstants;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.nodes.FormElement;\nimport org.jsoup.nodes.Node;\nimport org.jsoup.nodes.XmlDeclaration;\nimport org.jsoup.parser.HtmlTreeBuilder;\nimport org.jsoup.parser.Parser;\nimport org.jsoup.parser.StreamParser;\nimport org.jsoup.parser.XmlTreeBuilder;\nimport org.junit.jupiter.api.BeforeAll;\nimport org.junit.jupiter.api.BeforeEach;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.MethodSource;\nimport org.junit.jupiter.params.provider.ValueSource;\n\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.ByteArrayInputStream;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.net.Authenticator;\nimport java.net.MalformedURLException;\nimport java.net.URL;\nimport java.net.URLDecoder;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Stream;\n\nimport static org.jsoup.helper.AuthenticationHandlerTest.MaxAttempts;\nimport static org.jsoup.helper.HttpConnection.CONTENT_TYPE;\nimport static org.jsoup.helper.HttpConnection.MULTIPART_FORM_DATA;\nimport static org.junit.jupiter.api.Assertions.*;\n\n/**\n * Tests Jsoup.connect against a local server.\n */\npublic class ConnectTest {\n private static final int LargeDocFileLen = 280735;\n private static final int LargeDocTextLen = 269535;\n private static String echoUrl;\n\n @BeforeAll\n public static void setUp() {\n TestServer.start();\n echoUrl = EchoServlet.Url;\n System.setProperty(SharedConstants.UseHttpClient, \"false\"); // use the default UrlConnection. See HttpClientConnectTest for other version\n }\n\n @BeforeEach\n public void emptyCookieJar() {\n // empty the cookie jar, so cookie tests are independent.\n Jsoup.connect(\"http://example.com\").cookieStore().removeAll();\n }\n\n @Test\n public void canConnectToLocalServer() throws IOException {\n String url = HelloServlet.Url;\n Document doc = Jsoup.connect(url).get();\n Element p = doc.selectFirst(\"p\");\n assertEquals(\"Hello, World!\", p.text());\n }\n\n @Test void canConnectToLocalTlsServer() throws IOException {\n String url = HelloServlet.TlsUrl;\n Document doc = Jsoup.connect(url).get();\n Element p = doc.selectFirst(\"p\");\n assertEquals(\"Hello, World!\", p.text());\n }\n\n @Test\n public void fetchURl() throws IOException {\n Document doc = Jsoup.parse(new URL(echoUrl), 10 * 1000);\n assertTrue(doc.title().contains(\"Environment Variables\"));\n }\n\n @Test\n public void fetchURIWithWhitespace() throws IOException {\n Connection con = Jsoup.connect(echoUrl + \"#with whitespaces\");\n Document doc = con.get();\n assertTrue(doc.title().contains(\"Environment Variables\"));\n }\n\n @Test\n public void exceptOnUnsupportedProtocol() {\n String url = \"file://etc/passwd\";\n boolean threw = false;\n try {\n Document doc = Jsoup.connect(url).get();\n } catch (MalformedURLException e) {\n threw = true;\n assertEquals(\"java.net.MalformedURLException: Only http & https protocols supported\", e.toString());\n } catch (IOException e) {\n }\n assertTrue(threw);\n }\n\n static String ihVal(String key, Document doc) {\n final Element first = doc.select(\"th:contains(\" + key + \") + td\").first();\n return first != null ? first.text() : null;\n }\n\n @Test void statusMessage() throws IOException {\n Connection con = Jsoup.connect(EchoServlet.Url);\n Document doc = con.get();\n assertEquals(\"OK\", con.response().statusMessage());\n }\n\n @Test\n public void throwsExceptionOn404() {\n String url = EchoServlet.Url;\n Connection con = Jsoup.connect(url).header(EchoServlet.CodeParam, \"404\");\n\n boolean threw = false;\n try {\n Document doc = con.get();\n } catch (HttpStatusException e) {\n threw = true;\n assertEquals(\"org.jsoup.HttpStatusException: HTTP error fetching URL. Status=404, URL=[\" + e.getUrl() + \"]\", e.toString());\n assertTrue(e.getUrl().startsWith(url));\n assertEquals(404, e.getStatusCode());\n } catch (IOException e) {\n }\n assertTrue(threw);\n }\n\n @Test\n public void ignoresExceptionIfSoConfigured() throws IOException {\n String url = EchoServlet.Url;\n Connection con = Jsoup.connect(url)\n .header(EchoServlet.CodeParam, \"404\")\n .ignoreHttpErrors(true);\n Connection.Response res = con.execute();\n Document doc = res.parse();\n assertEquals(404, res.statusCode());\n assertEquals(\"Not Found\", res.statusMessage());\n assertEquals(\"Webserver Environment Variables\", doc.title());\n }\n\n @Test\n public void doesPost() throws IOException {\n Document doc = Jsoup.connect(echoUrl)\n .data(\"uname\", \"Jsoup\", \"uname\", \"Jonathan\", \"百\", \"度一下\")\n .cookie(\"auth\", \"token\")\n .post();\n\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"gzip\", ihVal(\"Accept-Encoding\", doc));\n assertEquals(\"auth=token\", ihVal(\"Cookie\", doc));\n assertEquals(\"度一下\", ihVal(\"百\", doc));\n assertEquals(\"Jsoup, Jonathan\", ihVal(\"uname\", doc));\n assertEquals(\"application/x-www-form-urlencoded; charset=UTF-8\", ihVal(\"Content-Type\", doc));\n }\n\n @Test\n public void doesPostMultipartWithoutInputstream() throws IOException {\n Document doc = Jsoup.connect(echoUrl)\n .header(CONTENT_TYPE, MULTIPART_FORM_DATA)\n .data(\"uname\", \"Jsoup\", \"uname\", \"Jonathan\", \"百\", \"度一下\")\n .post();\n\n assertTrue(ihVal(\"Content-Type\", doc).contains(MULTIPART_FORM_DATA));\n\n assertTrue(ihVal(\"Content-Type\", doc).contains(\"boundary\")); // should be automatically set\n assertEquals(\"Jsoup, Jonathan\", ihVal(\"uname\", doc));\n assertEquals(\"度一下\", ihVal(\"百\", doc));\n }\n\n @Test\n public void canSendSecFetchHeaders() throws IOException {\n // https://github.com/jhy/jsoup/issues/1461\n Document doc = Jsoup.connect(echoUrl)\n .header(\"Random-Header-name\", \"hello\")\n .header(\"Sec-Fetch-Site\", \"cross-site\")\n .header(\"Sec-Fetch-Mode\", \"cors\")\n .get();\n\n assertEquals(\"hello\", ihVal(\"Random-Header-name\", doc));\n assertEquals(\"cross-site\", ihVal(\"Sec-Fetch-Site\", doc));\n assertEquals(\"cors\", ihVal(\"Sec-Fetch-Mode\", doc));\n }\n\n @Test\n public void secFetchHeadersSurviveRedirect() throws IOException {\n Document doc = Jsoup\n .connect(RedirectServlet.Url)\n .data(RedirectServlet.LocationParam, echoUrl)\n .header(\"Random-Header-name\", \"hello\")\n .header(\"Sec-Fetch-Site\", \"cross-site\")\n .header(\"Sec-Fetch-Mode\", \"cors\")\n .get();\n\n assertEquals(\"hello\", ihVal(\"Random-Header-name\", doc));\n assertEquals(\"cross-site\", ihVal(\"Sec-Fetch-Site\", doc));\n assertEquals(\"cors\", ihVal(\"Sec-Fetch-Mode\", doc));\n }\n\n @Test\n public void sendsRequestBodyJsonWithData() throws IOException {\n final String body = \"{key:value}\";\n Document doc = Jsoup.connect(echoUrl)\n .requestBody(body)\n .header(\"Content-Type\", \"application/json\")\n .data(\"foo\", \"true\")\n .post();\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"application/json\", ihVal(\"Content-Type\", doc));\n assertEquals(\"foo=true\", ihVal(\"Query String\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @Test\n public void sendsRequestBodyJsonWithoutData() throws IOException {\n final String body = \"{key:value}\";\n Document doc = Jsoup.connect(echoUrl)\n .requestBody(body)\n .header(\"Content-Type\", \"application/json\")\n .post();\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"application/json\", ihVal(\"Content-Type\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @Test\n public void sendsRequestBody() throws IOException {\n final String body = \"{key:value}\";\n Document doc = Jsoup.connect(echoUrl)\n .requestBody(body)\n .header(\"Content-Type\", \"text/plain\")\n .post();\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"text/plain\", ihVal(\"Content-Type\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @Test\n public void sendsRequestBodyWithUrlParams() throws IOException {\n final String body = \"{key:value}\";\n Document doc = Jsoup.connect(echoUrl)\n .requestBody(body)\n .data(\"uname\", \"Jsoup\", \"uname\", \"Jonathan\", \"百\", \"度一下\")\n .header(\"Content-Type\", \"text/plain\") // todo - if user sets content-type, we should append postcharset\n .post();\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"uname=Jsoup&uname=Jonathan&%E7%99%BE=%E5%BA%A6%E4%B8%80%E4%B8%8B\", ihVal(\"Query String\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @Test void sendsRequestBodyStream() throws IOException {\n final String body = \"{key:value}\";\n InputStream stream = new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8));\n\n Document doc = Jsoup.connect(echoUrl)\n .requestBodyStream(stream)\n .header(\"Content-Type\", \"application/json\")\n .data(\"foo\", \"true\")\n .post();\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"application/json\", ihVal(\"Content-Type\", doc));\n assertEquals(\"foo=true\", ihVal(\"Query String\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\") // http and https\n public void doesGet(String url) throws IOException {\n Connection con = Jsoup.connect(url + \"?what=the\")\n .userAgent(\"Mozilla\")\n .referrer(\"http://example.com\")\n .data(\"what\", \"about & me?\");\n\n Document doc = con.get();\n assertEquals(\"what=the&what=about+%26+me%3F\", ihVal(\"Query String\", doc));\n assertEquals(\"the, about & me?\", ihVal(\"what\", doc));\n assertEquals(\"Mozilla\", ihVal(\"User-Agent\", doc));\n assertEquals(\"http://example.com\", ihVal(\"Referer\", doc));\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\") // http and https\n public void streamParserGet(String url) throws IOException {\n Connection con = Jsoup.connect(url)\n .userAgent(\"Mozilla\")\n .referrer(\"http://example.com\")\n .data(\"what\", \"about & me?\");\n\n //final Element first = doc.select(\"th:contains(\" + key + \") + td\").first();\n try (StreamParser streamer = con.execute().streamParser()) {\n Element title = streamer.expectFirst(\"title\");\n assertEquals(\"Webserver Environment Variables\", title.text());\n Element method = streamer.expectNext(echoSelect(\"Method\"));\n assertEquals(\"GET\", method.text());\n\n Document doc = streamer.document();\n assertSame(doc, title.ownerDocument());\n\n assertEquals(url + \"?what=about+%26+me%3F\", doc.location()); // with the query string\n }\n }\n\n static String echoSelect(String key) {\n return String.format(\"th:contains(%s) + td\", key);\n }\n\n @Test\n public void doesPut() throws IOException {\n Connection.Response res = Jsoup.connect(echoUrl)\n .data(\"uname\", \"Jsoup\", \"uname\", \"Jonathan\", \"百\", \"度一下\")\n .cookie(\"auth\", \"token\")\n .method(Connection.Method.PUT)\n .execute();\n\n Document doc = res.parse();\n assertEquals(\"PUT\", ihVal(\"Method\", doc));\n assertEquals(\"gzip\", ihVal(\"Accept-Encoding\", doc));\n assertEquals(\"auth=token\", ihVal(\"Cookie\", doc));\n }\n\n @Test\n public void doesDeleteWithBody() throws IOException {\n // https://github.com/jhy/jsoup/issues/1972\n String body = \"some body\";\n Connection.Response res = Jsoup.connect(echoUrl)\n .requestBody(body)\n .method(Method.DELETE)\n .execute();\n\n Document doc = res.parse();\n assertEquals(\"DELETE\", ihVal(\"Method\", doc));\n assertEquals(body, ihVal(\"Post Data\", doc));\n }\n\n @Test\n public void doesDeleteWithoutBody() throws IOException {\n Connection.Response res = Jsoup.connect(echoUrl)\n .method(Method.DELETE)\n .execute();\n\n Document doc = res.parse();\n assertEquals(\"DELETE\", ihVal(\"Method\", doc));\n assertEquals(null, ihVal(\"Post Data\", doc));\n }\n\n /**\n * Tests upload of content to a remote service.\n */\n @ParameterizedTest @MethodSource(\"echoUrls\") // http and https\n public void postFiles(String url) throws IOException {\n File thumb = ParseTest.getFile(\"/htmltests/thumb.jpg\");\n File html = ParseTest.getFile(\"/htmltests/large.html\");\n\n Document res = Jsoup\n .connect(url)\n .data(\"firstname\", \"Jay\")\n .data(\"firstPart\", thumb.getName(), Files.newInputStream(thumb.toPath()), \"image/jpeg\")\n .data(\"secondPart\", html.getName(), Files.newInputStream(html.toPath())) // defaults to \"application-octetstream\";\n .data(\"surname\", \"Soup\")\n .post();\n\n assertEquals(\"4\", ihVal(\"Parts\", res));\n\n assertEquals(\"application/octet-stream\", ihVal(\"Part secondPart ContentType\", res));\n assertEquals(\"secondPart\", ihVal(\"Part secondPart Name\", res));\n assertEquals(\"large.html\", ihVal(\"Part secondPart Filename\", res));\n assertEquals(\"280735\", ihVal(\"Part secondPart Size\", res));\n\n assertEquals(\"image/jpeg\", ihVal(\"Part firstPart ContentType\", res));\n assertEquals(\"firstPart\", ihVal(\"Part firstPart Name\", res));\n assertEquals(\"thumb.jpg\", ihVal(\"Part firstPart Filename\", res));\n assertEquals(\"1052\", ihVal(\"Part firstPart Size\", res));\n\n assertEquals(\"Jay\", ihVal(\"firstname\", res));\n assertEquals(\"Soup\", ihVal(\"surname\", res));\n\n /*\n <tr><th>Part secondPart ContentType</th><td>application/octet-stream</td></tr>\n <tr><th>Part secondPart Name</th><td>secondPart</td></tr>\n <tr><th>Part secondPart Filename</th><td>google-ipod.html</td></tr>\n <tr><th>Part secondPart Size</th><td>43972</td></tr>\n <tr><th>Part firstPart ContentType</th><td>image/jpeg</td></tr>\n <tr><th>Part firstPart Name</th><td>firstPart</td></tr>\n <tr><th>Part firstPart Filename</th><td>thumb.jpg</td></tr>\n <tr><th>Part firstPart Size</th><td>1052</td></tr>\n */\n }\n\n @Test\n public void multipleParsesOkAfterReadFully() throws IOException {\n Connection.Response res = Jsoup.connect(echoUrl).execute().readFully();\n\n Document doc = res.parse();\n assertTrue(doc.title().contains(\"Environment\"));\n\n Document doc2 = res.parse();\n assertTrue(doc2.title().contains(\"Environment\"));\n }\n\n @Test\n public void multipleParsesOkAfterBufferUp() throws IOException {\n Connection.Response res = Jsoup.connect(echoUrl).execute().bufferUp();\n\n Document doc = res.parse();\n assertTrue(doc.title().contains(\"Environment\"));\n\n Document doc2 = res.parse();\n assertTrue(doc2.title().contains(\"Environment\"));\n }\n\n @Test\n public void bodyAfterParseThrowsValidationError() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection.Response res = Jsoup.connect(echoUrl).execute();\n Document doc = res.parse();\n String body = res.body();\n });\n }\n\n @Test\n public void bodyAndBytesAvailableBeforeParse() throws IOException {\n Connection.Response res = Jsoup.connect(echoUrl).execute();\n String body = res.body();\n assertTrue(body.contains(\"Environment\"));\n byte[] bytes = res.bodyAsBytes();\n assertTrue(bytes.length > 100);\n\n Document doc = res.parse();\n assertTrue(doc.title().contains(\"Environment\"));\n }\n\n @Test\n public void parseParseThrowsValidates() {\n assertThrows(IllegalArgumentException.class, () -> {\n Connection.Response res = Jsoup.connect(echoUrl).execute();\n Document doc = res.parse();\n assertTrue(doc.title().contains(\"Environment\"));\n Document doc2 = res.parse(); // should blow up because the response input stream has been drained\n });\n }\n\n\n @Test\n public void multiCookieSet() throws IOException {\n Connection con = Jsoup\n .connect(RedirectServlet.Url)\n .data(RedirectServlet.CodeParam, \"302\")\n .data(RedirectServlet.SetCookiesParam, \"true\")\n .data(RedirectServlet.LocationParam, echoUrl);\n Connection.Response res = con.execute();\n\n // test cookies set by redirect:\n Map<String, String> cookies = res.cookies();\n assertEquals(\"asdfg123\", cookies.get(\"token\"));\n assertEquals(\"jhy\", cookies.get(\"uid\")); // two uids set, order dependent\n\n // send those cookies into the echo URL by map:\n Document doc = Jsoup.connect(echoUrl).cookies(cookies).get();\n assertEquals(\"token=asdfg123; uid=jhy\", ihVal(\"Cookie\", doc));\n }\n\n @Test public void requestCookiesSurviveRedirect() throws IOException {\n // this test makes sure that Request keyval cookies (not in the cookie store) are sent on subsequent redirections,\n // when not using the session method\n Connection con = Jsoup.connect(RedirectServlet.Url)\n .data(RedirectServlet.LocationParam, echoUrl)\n .cookie(\"LetMeIn\", \"True\")\n .cookie(\"DoesItWork\", \"Yes\");\n\n Connection.Response res = con.execute();\n assertEquals(0, res.cookies().size()); // were not set by Redir or Echo servlet\n Document doc = res.parse();\n assertEquals(echoUrl, doc.location());\n assertEquals(\"True\", ihVal(\"Cookie: LetMeIn\", doc));\n assertEquals(\"Yes\", ihVal(\"Cookie: DoesItWork\", doc));\n }\n\n @Test\n public void supportsDeflate() throws IOException {\n Connection.Response res = Jsoup.connect(DeflateServlet.Url).execute();\n assertEquals(\"deflate\", res.header(\"Content-Encoding\"));\n\n Document doc = res.parse();\n assertEquals(\"Hello, World!\", doc.selectFirst(\"p\").text());\n }\n\n @Test\n public void handlesLargerContentLengthParseRead() throws IOException {\n // this handles situations where the remote server sets a content length greater than it actually writes\n\n Connection.Response res = Jsoup.connect(InterruptedServlet.Url)\n .data(InterruptedServlet.Magnitude, InterruptedServlet.Larger)\n .timeout(400)\n .execute();\n\n try {\n Document document = res.parse();\n assertEquals(\"Something\", document.title());\n assertEquals(0, document.select(\"p\").size());\n } catch (IOException ignored) {\n // HttpUrlConnection will read the amount provided and the tests in the try will pass\n // HttpClient will throw unexpected EOF during the read, and this will catch it\n // Either are OK\n }\n // current impl, jetty won't write past content length\n // todo - find way to trick jetty into writing larger than set header. Take over the stream?\n }\n\n @Test\n public void handlesWrongContentLengthDuringBufferedRead() throws IOException {\n Connection.Response res = Jsoup.connect(InterruptedServlet.Url)\n .timeout(400)\n .execute();\n // this servlet writes max_buffer data, but sets content length to max_buffer/2. So will read up to that.\n // previous versions of jetty would allow to write less, and would throw except here\n\n res.bufferUp();\n Document doc = res.parse();\n assertEquals(0, doc.select(\"p\").size());\n }\n\n @Test public void handlesRedirect() throws IOException {\n Document doc = Jsoup.connect(RedirectServlet.Url)\n .data(RedirectServlet.LocationParam, HelloServlet.Url)\n .get();\n\n Element p = doc.selectFirst(\"p\");\n assertEquals(\"Hello, World!\", p.text());\n\n assertEquals(HelloServlet.Url, doc.location());\n }\n\n @Test public void handlesEmptyRedirect() {\n boolean threw = false;\n try {\n Connection.Response res = Jsoup.connect(RedirectServlet.Url)\n .execute();\n } catch (IOException e) {\n assertTrue(e.getMessage().contains(\"Too many redirects\"));\n threw = true;\n }\n assertTrue(threw);\n }\n\n @Test public void doesNotPostFor302() throws IOException {\n final Document doc = Jsoup.connect(RedirectServlet.Url)\n .data(\"Hello\", \"there\")\n .data(RedirectServlet.LocationParam, EchoServlet.Url)\n .post();\n\n assertEquals(EchoServlet.Url, doc.location());\n assertEquals(\"GET\", ihVal(\"Method\", doc));\n assertNull(ihVal(\"Hello\", doc)); // data not sent\n }\n\n @Test public void doesPostFor307() throws IOException {\n final Document doc = Jsoup.connect(RedirectServlet.Url)\n .data(\"Hello\", \"there\")\n .data(RedirectServlet.LocationParam, EchoServlet.Url)\n .data(RedirectServlet.CodeParam, \"307\")\n .post();\n\n assertEquals(EchoServlet.Url, doc.location());\n assertEquals(\"POST\", ihVal(\"Method\", doc));\n assertEquals(\"there\", ihVal(\"Hello\", doc));\n }\n\n @Test public void getUtf8Bom() throws IOException {\n Connection con = Jsoup.connect(FileServlet.urlTo(\"/bomtests/bom_utf8.html\"));\n Document doc = con.get();\n\n assertEquals(\"UTF-8\", con.response().charset());\n assertEquals(\"OK\", doc.title());\n }\n\n @Test public void streamerGetUtf8Bom() throws IOException {\n Connection con = Jsoup.connect(FileServlet.urlTo(\"/bomtests/bom_utf8.html\"));\n Document doc = con.execute().streamParser().complete();\n\n assertEquals(\"UTF-8\", con.response().charset());\n assertEquals(\"OK\", doc.title());\n }\n\n @Test\n public void testBinaryContentTypeThrowsException() throws IOException {\n Connection con = Jsoup.connect(FileServlet.urlTo(\"/htmltests/thumb.jpg\"));\n con.data(FileServlet.ContentTypeParam, \"image/jpeg\");\n\n boolean threw = false;\n try {\n con.execute();\n Document doc = con.response().parse();\n } catch (UnsupportedMimeTypeException e) {\n threw = true;\n assertEquals(\"Unhandled content type. Must be text/*, */xml, or */*+xml\", e.getMessage());\n }\n assertTrue(threw);\n }\n\n @Test public void testParseRss() throws IOException {\n // test that we switch automatically to xml, and we support application/rss+xml\n Connection con = Jsoup.connect(FileServlet.urlTo(\"/htmltests/test-rss.xml\"));\n con.data(FileServlet.ContentTypeParam, \"application/rss+xml\");\n Document doc = con.get();\n Element title = doc.selectFirst(\"title\");\n assertNotNull(title);\n assertEquals(\"jsoup RSS news\", title.text());\n assertEquals(\"channel\", title.parent().nodeName());\n assertEquals(\"\", doc.title()); // the document title is unset, this tag is channel>title, not html>head>title\n assertEquals(3, doc.select(\"link\").size());\n assertEquals(\"application/rss+xml\", con.response().contentType());\n assertTrue(doc.parser().getTreeBuilder() instanceof XmlTreeBuilder);\n assertEquals(Document.OutputSettings.Syntax.xml, doc.outputSettings().syntax());\n }\n\n @Test\n public void testSupplyParserToConnection() throws IOException {\n String xmlUrl = FileServlet.urlTo(\"/htmltests/xml-test.xml\");\n\n // parse with both xml and html parser, ensure different\n Document xmlDoc = Jsoup.connect(xmlUrl).parser(Parser.xmlParser()).get();\n Document htmlDoc = Jsoup.connect(xmlUrl).parser(Parser.htmlParser()).get();\n Document autoXmlDoc = Jsoup.connect(xmlUrl).get(); // check connection auto detects xml, uses xml parser\n\n assertEquals(\"<doc><val>One<val>Two</val>Three</val></doc>\",\n TextUtil.stripNewlines(xmlDoc.html()));\n assertNotEquals(htmlDoc, xmlDoc);\n assertFalse(htmlDoc.hasSameValue(xmlDoc));\n assertTrue(xmlDoc.hasSameValue(autoXmlDoc));\n\n assertEquals(1, htmlDoc.select(\"head\").size()); // html parser normalises\n assertEquals(0, xmlDoc.select(\"head\").size()); // xml parser does not\n assertEquals(0, autoXmlDoc.select(\"head\").size()); // xml parser does not\n }\n\n @Test public void imageXmlMimeType() throws IOException {\n // test that we switch to XML, and that we support image/svg+xml\n String mimetype = \"image/svg+xml\";\n\n Connection con = Jsoup.connect(FileServlet.urlTo(\"/htmltests/osi-logo.svg\"))\n .data(FileServlet.ContentTypeParam, mimetype);\n Document doc = con.get();\n\n assertEquals(mimetype, con.response().contentType());\n assertTrue(doc.parser().getTreeBuilder() instanceof XmlTreeBuilder);\n assertEquals(Document.OutputSettings.Syntax.xml, doc.outputSettings().syntax());\n Node firstChild = doc.firstChild();\n XmlDeclaration decl = (XmlDeclaration) firstChild;\n assertEquals(\"no\", decl.attr(\"standalone\"));\n Element svg = doc.expectFirst(\"svg\");\n Element flowRoot = svg.expectFirst(\"flowRoot\");\n assertEquals(\"flowRoot\", flowRoot.tagName());\n assertEquals(\"preserve\", flowRoot.attr(\"xml:space\"));\n }\n\n @Test\n public void canFetchBinaryAsBytes() throws IOException {\n String path = \"/htmltests/thumb.jpg\";\n int actualSize = 1052;\n\n Connection.Response res = Jsoup.connect(FileServlet.urlTo(path))\n .data(FileServlet.ContentTypeParam, \"image/jpeg\")\n .ignoreContentType(true)\n .execute();\n\n byte[] resBytes = res.bodyAsBytes();\n assertEquals(actualSize, resBytes.length);\n\n // compare the content of the file and the bytes:\n Path filePath = ParseTest.getPath(path);\n byte[] fileBytes = Files.readAllBytes(filePath);\n assertEquals(actualSize, fileBytes.length);\n assertArrayEquals(fileBytes, resBytes);\n }\n\n @Test\n public void handlesUnknownEscapesAcrossBuffer() throws IOException {\n String localPath = \"/htmltests/escapes-across-buffer.html\";\n String localUrl = FileServlet.urlTo(localPath);\n\n Document docFromLocalServer = Jsoup.connect(localUrl).get();\n Document docFromFileRead = Jsoup.parse(ParseTest.getFile(localPath), \"UTF-8\");\n\n String text = docFromLocalServer.body().text();\n assertEquals(14766, text.length());\n assertEquals(text, docFromLocalServer.body().text());\n assertEquals(text, docFromFileRead.body().text());\n }\n\n /**\n * Test fetching a form, and submitting it with a file attached.\n */\n @Test\n public void postHtmlFile() throws IOException {\n Document index = Jsoup.connect(FileServlet.urlTo(\"/htmltests/upload-form.html\")).get();\n List<FormElement> forms = index.select(\"[name=tidy]\").forms();\n assertEquals(1, forms.size());\n FormElement form = forms.get(0);\n Connection post = form.submit();\n\n File uploadFile = ParseTest.getFile(\"/htmltests/large.html\");\n FileInputStream stream = new FileInputStream(uploadFile);\n\n Connection.KeyVal fileData = post.data(\"_file\");\n assertNotNull(fileData);\n fileData.value(\"check.html\");\n fileData.inputStream(stream);\n\n Connection.Response res;\n try {\n res = post.execute();\n } finally {\n stream.close();\n }\n\n Document doc = res.parse();\n assertEquals(ihVal(\"Method\", doc), \"POST\"); // from form action\n assertEquals(ihVal(\"Part _file Filename\", doc), \"check.html\");\n assertEquals(ihVal(\"Part _file Name\", doc), \"_file\");\n assertEquals(ihVal(\"_function\", doc), \"tidy\");\n }\n\n @Test\n public void fetchHandlesXml() throws IOException {\n String[] types = {\"text/xml\", \"application/xml\", \"application/rss+xml\", \"application/xhtml+xml\"};\n for (String type : types) {\n fetchHandlesXml(type);\n }\n }\n\n void fetchHandlesXml(String contentType) throws IOException {\n // should auto-detect xml and use XML parser, unless explicitly requested the html parser\n String xmlUrl = FileServlet.urlTo(\"/htmltests/xml-test.xml\");\n Connection con = Jsoup.connect(xmlUrl);\n con.data(FileServlet.ContentTypeParam, contentType);\n Document doc = con.get();\n Connection.Request req = con.request();\n assertTrue(req.parser().getTreeBuilder() instanceof XmlTreeBuilder);\n assertEquals(\"<doc><val>One<val>Two</val>Three</val></doc>\\n\", doc.outerHtml());\n assertEquals(con.response().contentType(), contentType);\n }\n\n @Test\n public void fetchHandlesXmlAsHtmlWhenParserSet() throws IOException {\n // should auto-detect xml and use XML parser, unless explicitly requested the html parser\n String xmlUrl = FileServlet.urlTo(\"/htmltests/xml-test.xml\");\n Connection con = Jsoup.connect(xmlUrl).parser(Parser.htmlParser()); // which will also use the pretty printer by default\n con.data(FileServlet.ContentTypeParam, \"application/xml\");\n Document doc = con.get();\n Connection.Request req = con.request();\n assertTrue(req.parser().getTreeBuilder() instanceof HtmlTreeBuilder);\n assertEquals(\"<html>\\n <head></head>\\n <body>\\n <doc>\\n <val>\\n One<val>Two</val>Three\\n </val>\\n </doc>\\n </body>\\n</html>\", doc.outerHtml());\n }\n\n @Test\n public void combinesSameHeadersWithComma() throws IOException {\n // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2\n Connection con = Jsoup.connect(echoUrl);\n con.get();\n\n Connection.Response res = con.response();\n assertEquals(\"text/html;charset=utf-8\", res.header(\"Content-Type\"));\n assertEquals(\"no-cache, no-store\", res.header(\"Cache-Control\"));\n\n List<String> header = res.headers(\"Cache-Control\");\n assertEquals(2, header.size());\n assertEquals(\"no-cache\", header.get(0));\n assertEquals(\"no-store\", header.get(1));\n }\n\n @Test\n public void sendHeadRequest() throws IOException {\n String url = FileServlet.urlTo(\"/htmltests/xml-test.xml\");\n Connection con = Jsoup.connect(url)\n .method(Connection.Method.HEAD)\n .data(FileServlet.ContentTypeParam, \"text/xml\");\n final Connection.Response response = con.execute();\n assertEquals(\"text/xml\", response.header(\"Content-Type\"));\n assertEquals(\"\", response.body()); // head ought to have no body\n Document doc = response.parse();\n assertEquals(\"\", doc.text());\n }\n\n @Test\n public void fetchToW3c() throws IOException {\n String url = FileServlet.urlTo(\"/htmltests/upload-form.html\");\n Document doc = Jsoup.connect(url).get();\n\n W3CDom dom = new W3CDom();\n org.w3c.dom.Document wDoc = dom.fromJsoup(doc);\n assertEquals(url, wDoc.getDocumentURI());\n String html = dom.asString(wDoc);\n assertTrue(html.contains(\"Upload\"));\n }\n\n @Test\n public void baseHrefCorrectAfterHttpEquiv() throws IOException {\n // https://github.com/jhy/jsoup/issues/440\n Connection.Response res = Jsoup.connect(FileServlet.urlTo(\"/htmltests/charset-base.html\")).execute();\n Document doc = res.parse();\n assertEquals(\"http://example.com/foo.jpg\", doc.select(\"img\").first().absUrl(\"src\"));\n }\n\n @Test\n public void maxBodySize() throws IOException {\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n\n Connection.Response defaultRes = Jsoup.connect(url).execute();\n Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops\n Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops\n Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop\n Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();\n\n int actualDocText = LargeDocTextLen;\n assertEquals(actualDocText, defaultRes.parse().text().length());\n assertEquals(49165, smallRes.parse().text().length());\n assertEquals(196577, mediumRes.parse().text().length());\n assertEquals(actualDocText, largeRes.parse().text().length());\n assertEquals(actualDocText, unlimitedRes.parse().text().length());\n }\n\n @Test public void repeatable() throws IOException {\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n Connection con = Jsoup.connect(url).parser(Parser.xmlParser());\n Document doc1 = con.get();\n Document doc2 = con.get();\n assertEquals(\"Large HTML\", doc1.title());\n assertEquals(\"Large HTML\", doc2.title());\n }\n\n @Test\n public void maxBodySizeInReadToByteBuffer() throws IOException {\n // https://github.com/jhy/jsoup/issues/1774\n // when calling readToByteBuffer, contents were not buffered up\n String url = FileServlet.urlTo(\"/htmltests/large.html\"); // 280 K\n\n Connection.Response defaultRes = Jsoup.connect(url).execute();\n Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops\n Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops\n Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop\n Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();\n\n int actualDocText = 280735;\n assertEquals(actualDocText, defaultRes.body().length());\n assertEquals(50 * 1024, smallRes.body().length());\n assertEquals(200 * 1024, mediumRes.body().length());\n assertEquals(actualDocText, largeRes.body().length());\n assertEquals(actualDocText, unlimitedRes.body().length());\n\n assertEquals(actualDocText, defaultRes.readBody().length());\n assertEquals(50 * 1024, smallRes.readBody().length());\n assertEquals(200 * 1024, mediumRes.readBody().length());\n assertEquals(actualDocText, largeRes.readBody().length());\n assertEquals(actualDocText, unlimitedRes.readBody().length());\n }\n\n @Test void formLoginFlow() throws IOException {\n String echoUrl = EchoServlet.Url;\n String cookieUrl = CookieServlet.Url;\n\n String startUrl = FileServlet.urlTo(\"/htmltests/form-tests.html\");\n Document loginDoc = Jsoup.connect(startUrl).get();\n FormElement form = loginDoc.expectForm(\"#login\");\n assertNotNull(form);\n form.expectFirst(\"[name=username]\").val(\"admin\");\n form.expectFirst(\"[name=password]\").val(\"Netscape engineers are weenies!\");\n\n // post it- should go to Cookie then bounce to Echo\n Connection submit = form.submit();\n assertEquals(Connection.Method.POST, submit.request().method());\n Connection.Response postRes = submit.execute();\n assertEquals(echoUrl, postRes.url().toExternalForm());\n assertEquals(Connection.Method.GET, postRes.method());\n Document resultDoc = postRes.parse();\n assertEquals(\"One=EchoServlet; One=Root\", ihVal(\"Cookie\", resultDoc));\n // should be no form data sent to the echo redirect\n assertEquals(\"\", ihVal(\"Query String\", resultDoc));\n\n // new request to echo, should not have form data, but should have cookies from implicit session\n Document newEcho = submit.newRequest(echoUrl).get();\n assertEquals(\"One=EchoServlet; One=Root\", ihVal(\"Cookie\", newEcho));\n assertEquals(\"\", ihVal(\"Query String\", newEcho));\n\n Document cookieDoc = submit.newRequest(cookieUrl).get();\n assertEquals(\"CookieServlet\", ihVal(\"One\", cookieDoc)); // different cookie path\n\n }\n\n @Test void formLoginFlow2() throws IOException {\n String echoUrl = EchoServlet.Url;\n String cookieUrl = CookieServlet.Url;\n String startUrl = FileServlet.urlTo(\"/htmltests/form-tests.html\");\n\n Connection session = Jsoup.newSession();\n Document loginDoc = session.newRequest(startUrl).get();\n FormElement form = loginDoc.expectForm(\"#login2\");\n assertNotNull(form);\n String username = \"admin\";\n form.expectFirst(\"[name=username]\").val(username);\n String password = \"Netscape engineers are weenies!\";\n form.expectFirst(\"[name=password]\").val(password);\n\n Connection submit = form.submit();\n assertEquals(username, submit.data(\"username\").value());\n assertEquals(password, submit.data(\"password\").value());\n\n Connection.Response postRes = submit.execute();\n assertEquals(cookieUrl, postRes.url().toExternalForm());\n assertEquals(Connection.Method.POST, postRes.method());\n Document resultDoc = postRes.parse();\n\n Document echo2 = resultDoc.connection().newRequest(echoUrl).get();\n assertEquals(\"\", ihVal(\"Query String\", echo2)); // should not re-send the data\n assertEquals(\"One=EchoServlet; One=Root\", ihVal(\"Cookie\", echo2));\n }\n\n @Test void preservesUrlFragment() throws IOException {\n // confirms https://github.com/jhy/jsoup/issues/1686\n String url = EchoServlet.Url + \"#fragment\";\n Document doc = Jsoup.connect(url).get();\n assertEquals(url, doc.location());\n }\n\n @Test void fetchUnicodeUrl() throws IOException {\n String url = EchoServlet.Url + \"/✔/?鍵=値\";\n Document doc = Jsoup.connect(url).get();\n\n assertEquals(\"/✔/\", ihVal(\"Path Info\", doc));\n assertEquals(\"%E9%8D%B5=%E5%80%A4\", ihVal(\"Query String\", doc));\n assertEquals(\"鍵=値\", URLDecoder.decode(ihVal(\"Query String\", doc), DataUtil.UTF_8.name()));\n }\n\n @Test void willEscapePathInRedirect() throws IOException {\n String append = \"/Foo{bar}<>%/\";\n String url = EchoServlet.Url + append;\n Document doc = Jsoup\n .connect(RedirectServlet.Url)\n .data(RedirectServlet.LocationParam, url)\n .get();\n\n String path = ihVal(\"Path Info\", doc);\n assertEquals(append, path);\n assertEquals(\"/EchoServlet/Foo%7Bbar%7D%3C%3E%25/\", ihVal(\"Request URI\", doc));\n }\n\n /**\n Provides HTTP and HTTPS EchoServlet URLs\n */\n private static Stream<String> echoUrls() {\n return Stream.of(EchoServlet.Url, EchoServlet.TlsUrl);\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\")\n void failsIfNotAuthenticated(String url) throws IOException {\n String password = AuthFilter.newServerPassword(); // we don't send it, but ensures cache won't hit\n Connection.Response res = Jsoup.connect(url)\n .header(AuthFilter.WantsServerAuthentication, \"1\")\n .ignoreHttpErrors(true)\n .execute();\n\n assertEquals(401, res.statusCode());\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\")\n void canAuthenticate(String url) throws IOException {\n AtomicInteger count = new AtomicInteger(0);\n String password = AuthFilter.newServerPassword();\n Connection.Response res = Jsoup.connect(url)\n .header(AuthFilter.WantsServerAuthentication, \"1\")\n .auth(ctx -> {\n count.incrementAndGet();\n assertEquals(Authenticator.RequestorType.SERVER, ctx.type());\n assertEquals(\"localhost\", ctx.url().getHost());\n assertEquals(AuthFilter.ServerRealm, ctx.realm());\n\n return ctx.credentials(AuthFilter.ServerUser, password);\n })\n .execute();\n\n assertEquals(1, count.get());\n\n Document doc = res.parse();\n assertTrue(ihVal(\"Authorization\", doc).startsWith(\"Basic \")); // tests we set the auth header\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\")\n void incorrectAuth(String url) throws IOException {\n Connection session = Jsoup.newSession()\n .header(AuthFilter.WantsServerAuthentication, \"1\")\n .ignoreHttpErrors(true);\n\n String password = AuthFilter.newServerPassword();\n int code = session.newRequest(url).execute().statusCode(); // no auth sent\n assertEquals(HttpServletResponse.SC_UNAUTHORIZED, code);\n\n AtomicInteger count = new AtomicInteger(0);\n try {\n Connection.Response res = session.newRequest(url)\n .auth(ctx -> {\n count.incrementAndGet();\n return ctx.credentials(AuthFilter.ServerUser, password + \"wrong\"); // incorrect\n })\n .execute();\n assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.statusCode());\n } catch (IOException e) {\n assertEquals(\"No credentials provided\", e.getMessage());\n // In HttpClient, will throw IOE if our password delegate stops providing credentials after too many attempts. So we'll get this error.\n // In HttpUrlConnection, which would otherwise try 20 times, when the auth stops providing it will cascade to the underyling 401 response (which seems a better path IMO)\n }\n assertEquals(MaxAttempts, count.get());\n\n AtomicInteger successCount = new AtomicInteger(0);\n Connection.Response successRes = session.newRequest(url)\n .auth(ctx -> {\n successCount.incrementAndGet();\n return ctx.credentials(AuthFilter.ServerUser, password); // correct\n })\n .execute();\n assertEquals(1, successCount.get());\n assertEquals(HttpServletResponse.SC_OK, successRes.statusCode());\n }\n\n // proxy connection tests are in ProxyTest\n\n @ParameterizedTest\n @ValueSource(strings = {\n \"/htmltests/large.html\",\n \"/htmltests/large.html?\" + FileServlet.SuppressContentLength\n })\n void progressListener(String path) throws IOException {\n String url = FileServlet.urlTo(path);\n boolean knownContentLength = !url.contains(FileServlet.SuppressContentLength);\n\n AtomicBoolean seenProgress = new AtomicBoolean(false);\n AtomicBoolean completed = new AtomicBoolean(false);\n AtomicInteger numProgress = new AtomicInteger();\n\n Connection con = Jsoup.connect(url).onResponseProgress((processed, total, percent, response) -> {\n //System.out.println(\"Processed: \" + processed + \" of \" + total + \" (\" + percent + \"%)\");\n if (!seenProgress.get()) {\n seenProgress.set(true);\n assertEquals(0, processed);\n assertEquals(knownContentLength ? LargeDocFileLen : -1, total);\n assertEquals(0.0f, percent);\n\n assertEquals(200, response.statusCode());\n String contentLength = response.header(\"Content-Length\");\n if (knownContentLength) {\n assertNotNull(contentLength);\n assertEquals(String.valueOf(LargeDocFileLen), contentLength);\n } else {\n assertNull(contentLength);\n }\n assertEquals(url, response.url().toExternalForm());\n }\n numProgress.getAndIncrement();\n\n if (percent == 100.0f) {\n // even if the content-length is not set, we get 100% when the read is completed\n completed.set(true);\n assertEquals(LargeDocFileLen, processed);\n }\n\n });\n Document document = con.get();\n\n assertTrue(seenProgress.get());\n assertTrue(completed.get());\n\n // should expect to see events relative to how large the buffer is.\n int expected = LargeDocFileLen / 8192;\n\n int num = numProgress.get();\n // debug log if not in those ranges:\n if (num < expected * 0.75 || num > expected * 2.5) {\n System.err.println(\"Expected: \" + expected + \", got: \" + num);\n }\n assertTrue(num > expected * 0.75);\n assertTrue(num < expected * 2.5);\n\n // check the document works\n assertEquals(LargeDocTextLen, document.text().length());\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/FuzzFixesIT.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Assertions;\nimport org.junit.jupiter.api.Disabled;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.MethodSource;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.stream.Stream;\n\nimport static org.junit.jupiter.api.Assertions.assertNotNull;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\n\n/**\n Tests fixes for issues raised by the <a href=\"https://oss-fuzz.com/testcases?project=jsoup\">OSS Fuzz project</a>. As\n some of these are timeout tests - run each file 100 times and ensure under time.\n */\npublic class FuzzFixesIT {\n static int numIters = 50;\n static int timeout = 30; // external fuzzer is set to 60 for 100 runs\n static File testDir = ParseTest.getFile(\"/fuzztests/\");\n\n private static Stream<File> testFiles() {\n File[] files = testDir.listFiles();\n assertNotNull(files);\n assertTrue(files.length > 10);\n\n return Stream.of(files);\n }\n\n @Disabled // disabled, as these soak up build time and the outcome oughtn't change unless we are refactoring the tree builders. manually execute as desired.\n @ParameterizedTest\n @MethodSource(\"testFiles\")\n void testHtmlParse(File file) throws IOException {\n long startTime = System.currentTimeMillis();\n long completeBy = startTime + timeout * 1000L;\n\n for (int i = 0; i < numIters; i++) {\n Document doc = Jsoup.parse(file, \"UTF-8\", \"https://example.com/\");\n assertNotNull(doc);\n if (System.currentTimeMillis() > completeBy)\n Assertions.fail(String.format(\"Timeout: only completed %d iters of [%s] in %d seconds\", i, file.getName(), timeout));\n }\n }\n \n @Disabled // disabled, as these soak up build time and the outcome oughtn't change unless we are refactoring the tree builders. manually execute as desired.\n @ParameterizedTest\n @MethodSource(\"testFiles\")\n void testXmlParse(File file) throws IOException {\n long startTime = System.currentTimeMillis();\n long completeBy = startTime + timeout * 1000L;\n\n for (int i = 0; i < numIters; i++) {\n Document doc = Jsoup.parse(file, \"UTF-8\", \"https://example.com/\", Parser.xmlParser());\n assertNotNull(doc);\n if (System.currentTimeMillis() > completeBy)\n Assertions.fail(String.format(\"Timeout: only completed %d iters of [%s] in %d seconds\", i, file.getName(), timeout));\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/FuzzFixesTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.MethodSource;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.stream.Stream;\n\nimport static org.junit.jupiter.api.Assertions.assertNotNull;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\n\n/**\n Tests fixes for issues raised by the OSS Fuzz project @ https://oss-fuzz.com/testcases?project=jsoup. Contains inline\n string cases causing exceptions. Timeout tests are in FuzzFixesIT.\n */\npublic class FuzzFixesTest {\n\n private static Stream<File> testFiles() {\n File[] files = FuzzFixesIT.testDir.listFiles();\n assertNotNull(files);\n assertTrue(files.length > 10);\n\n return Stream.of(files);\n }\n\n @Test\n public void blankAbsAttr() {\n // https://github.com/jhy/jsoup/issues/1541\n String html = \"b<bodY abs: abs:abs: abs:abs:abs>\";\n Document doc = Jsoup.parse(html);\n assertNotNull(doc);\n }\n\n @Test\n public void bookmark() {\n // https://github.com/jhy/jsoup/issues/1576\n String html = \"<?a<U<P<A \";\n Document doc = Jsoup.parse(html);\n assertNotNull(doc);\n\n Document xmlDoc = Parser.xmlParser().parseInput(html, \"\");\n assertNotNull(xmlDoc);\n }\n\n @Test void fragment() {\n Parser.htmlParser().parseFragmentInput(\"<frameset>>l\\u0000<\\u0000<ditl>\\u0000< \\\\\", new Element(\"colgroup\"), \"\");\n }\n\n @ParameterizedTest\n @MethodSource(\"testFiles\")\n void testHtmlParse(File file) throws IOException {\n Document doc = Jsoup.parse(file, \"UTF-8\", \"https://example.com/\");\n assertNotNull(doc);\n doc = Jsoup.parse(file, \"UTF-8\", \"\"); // no base href attr; so same as a parse(string), which can have subtly different semantics\n assertNotNull(doc);\n }\n\n @ParameterizedTest\n @MethodSource(\"testFiles\")\n void testHtmlFragmentParse(File file) throws IOException {\n String html = ParseTest.getFileAsString(file);\n Document doc = Jsoup.parseBodyFragment(html);\n assertNotNull(doc);\n }\n\n @ParameterizedTest\n @MethodSource(\"testFiles\")\n void testXmlParse(File file) throws IOException {\n Document doc = Jsoup.parse(file, \"UTF-8\", \"https://example.com/\", Parser.xmlParser());\n assertNotNull(doc);\n doc = Jsoup.parse(file, \"UTF-8\", \"\", Parser.xmlParser()); // no base href attr\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/ParseTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.helper.DataUtil;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.parser.ParseErrorList;\nimport org.jsoup.parser.Parser;\nimport org.jsoup.select.Elements;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.*;\nimport java.net.URISyntaxException;\nimport java.net.URL;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.zip.GZIPInputStream;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\n/**\n * Integration test: parses from real-world example HTML.\n *\n * @author Jonathan Hedley, jonathan@hedley.net\n */\npublic class ParseTest {\n @Test\n public void testHtml5Charset() throws IOException {\n // test that <meta charset=\"gb2312\"> works\n File in = getFile(\"/htmltests/meta-charset-1.html\");\n Document doc = Jsoup.parse(in, null, \"http://example.com/\"); //gb2312, has html5 <meta charset>\n assertEquals(\"新\", doc.text());\n assertEquals(\"GB2312\", doc.outputSettings().charset().displayName());\n\n // double check, no charset, falls back to utf8 which is incorrect\n in = getFile(\"/htmltests/meta-charset-2.html\"); //\n doc = Jsoup.parse(in, null, \"http://example.com\"); // gb2312, no charset\n assertEquals(\"UTF-8\", doc.outputSettings().charset().displayName());\n assertNotEquals(\"新\", doc.text());\n\n // confirm fallback to utf8\n in = getFile(\"/htmltests/meta-charset-3.html\");\n doc = Jsoup.parse(in, null, \"http://example.com/\"); // utf8, no charset\n assertEquals(\"UTF-8\", doc.outputSettings().charset().displayName());\n assertEquals(\"新\", doc.text());\n }\n\n @Test\n public void testBrokenHtml5CharsetWithASingleDoubleQuote() throws IOException {\n InputStream in = inputStreamFrom(\"<html>\\n\" +\n \"<head><meta charset=UTF-8\\\"></head>\\n\" +\n \"<body></body>\\n\" +\n \"</html>\");\n Document doc = Jsoup.parse(in, null, \"http://example.com/\");\n assertEquals(\"UTF-8\", doc.outputSettings().charset().displayName());\n }\n\n @Test\n public void testLowercaseUtf8Charset() throws IOException {\n File in = getFile(\"/htmltests/lowercase-charset-test.html\");\n Document doc = Jsoup.parse(in, null);\n\n Element form = doc.select(\"#form\").first();\n assertEquals(2, form.children().size());\n assertEquals(\"UTF-8\", doc.outputSettings().charset().name());\n }\n\n @Test\n public void testXwiki() throws IOException {\n // https://github.com/jhy/jsoup/issues/1324\n // this tests that when in CharacterReader we hit a buffer while marked, we preserve the mark when buffered up and can rewind\n File in = getFile(\"/htmltests/xwiki-1324.html.gz\");\n Document doc = Jsoup.parse(in, null, \"https://localhost/\");\n assertEquals(\"XWiki Jetty HSQLDB 12.1-SNAPSHOT\", doc.select(\"#xwikiplatformversion\").text());\n\n // was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so\n // updated to preserve the mark.\n String wantHtml = \"<a class=\\\"list-group-item\\\" data-id=\\\"userdirectory\\\" href=\\\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin§ion=userdirectory\\\" title=\\\"Customize the user directory live table.\\\">User Directory</a>\";\n assertEquals(wantHtml, doc.select(\"[data-id=userdirectory]\").outerHtml());\n }\n\n @Test\n public void testXwikiExpanded() throws IOException {\n // https://github.com/jhy/jsoup/issues/1324\n // this tests that if there is a huge illegal character reference, we can get through a buffer and rewind, and still catch that it's an invalid refence,\n // and the parse tree is correct.\n File in = getFile(\"/htmltests/xwiki-edit.html.gz\");\n Parser parser = Parser.htmlParser();\n Document doc = Jsoup.parse(new GZIPInputStream(new FileInputStream(in)), \"UTF-8\", \"https://localhost/\", parser.setTrackErrors(100));\n ParseErrorList errors = parser.getErrors();\n\n assertEquals(\"XWiki Jetty HSQLDB 12.1-SNAPSHOT\", doc.select(\"#xwikiplatformversion\").text());\n assertEquals(0, errors.size()); // not an invalid reference because did not look legit\n\n // was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so\n // updated to preserve the mark.\n String wantHtml = \"<a class=\\\"list-group-item\\\" data-id=\\\"userdirectory\\\" href=\\\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin&RIGHTHERERIGHTHERERIGHTHERERIGHTHERE\";\n assertTrue(doc.select(\"[data-id=userdirectory]\").outerHtml().startsWith(wantHtml));\n }\n\n @Test public void testWikiExpandedFromString() throws IOException {\n File in = getFile(\"/htmltests/xwiki-edit.html.gz\");\n String html = getFileAsString(in);\n Document doc = Jsoup.parse(html);\n assertEquals(\"XWiki Jetty HSQLDB 12.1-SNAPSHOT\", doc.select(\"#xwikiplatformversion\").text());\n String wantHtml = \"<a class=\\\"list-group-item\\\" data-id=\\\"userdirectory\\\" href=\\\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin&RIGHTHERERIGHTHERERIGHTHERERIGHTHERE\";\n assertTrue(doc.select(\"[data-id=userdirectory]\").outerHtml().startsWith(wantHtml));\n }\n\n @Test public void testWikiFromString() throws IOException {\n File in = getFile(\"/htmltests/xwiki-1324.html.gz\");\n String html = getFileAsString(in);\n Document doc = Jsoup.parse(html);\n assertEquals(\"XWiki Jetty HSQLDB 12.1-SNAPSHOT\", doc.select(\"#xwikiplatformversion\").text());\n String wantHtml = \"<a class=\\\"list-group-item\\\" data-id=\\\"userdirectory\\\" href=\\\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin§ion=userdirectory\\\" title=\\\"Customize the user directory live table.\\\">User Directory</a>\";\n assertEquals(wantHtml, doc.select(\"[data-id=userdirectory]\").outerHtml());\n }\n\n @Test public void testFileParseNoCharsetMethod() throws IOException {\n File in = getFile(\"/htmltests/xwiki-1324.html.gz\");\n Document doc = Jsoup.parse(in);\n assertEquals(\"XWiki Jetty HSQLDB 12.1-SNAPSHOT\", doc.select(\"#xwikiplatformversion\").text());\n }\n\n\n public static File getFile(String resourceName) {\n try {\n URL resource = ParseTest.class.getResource(resourceName);\n return resource != null ? new File(resource.toURI()) : new File(\"/404\");\n } catch (URISyntaxException e) {\n throw new IllegalStateException(e);\n }\n }\n\n public static Path getPath(String resourceName) {\n try {\n URL resource = ParseTest.class.getResource(resourceName);\n return resource != null ? Paths.get(resource.toURI()) : Paths.get(\"/404\");\n } catch (URISyntaxException e) {\n throw new IllegalStateException(e);\n }\n }\n\n public static InputStream inputStreamFrom(String s) {\n return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));\n }\n\n public static String getFileAsString(File file) throws IOException {\n byte[] bytes;\n if (file.getName().endsWith(\".gz\")) {\n InputStream stream = new GZIPInputStream(new FileInputStream(file));\n ByteBuffer byteBuffer = DataUtil.readToByteBuffer(stream, 0);\n bytes = new byte[byteBuffer.limit()];\n System.arraycopy(byteBuffer.array(), 0, bytes, 0, byteBuffer.limit());\n } else {\n bytes = Files.readAllBytes(file.toPath());\n }\n return new String(bytes);\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/ProxyTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Connection;\nimport org.jsoup.Jsoup;\nimport org.jsoup.integration.servlets.AuthFilter;\nimport org.jsoup.integration.servlets.EchoServlet;\nimport org.jsoup.integration.servlets.FileServlet;\nimport org.jsoup.integration.servlets.HelloServlet;\nimport org.jsoup.integration.servlets.ProxyServlet;\nimport org.jsoup.integration.servlets.RedirectServlet;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.junit.jupiter.api.BeforeAll;\nimport org.junit.jupiter.api.Test;\nimport org.junit.jupiter.params.ParameterizedTest;\nimport org.junit.jupiter.params.provider.MethodSource;\n\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Stream;\n\nimport static org.jsoup.helper.AuthenticationHandlerTest.MaxAttempts;\nimport static org.jsoup.integration.ConnectTest.ihVal;\nimport static org.junit.jupiter.api.Assertions.*;\n\n/**\n Tests Jsoup.connect proxy support */\npublic class ProxyTest {\n private static String echoUrl;\n private static TestServer.ProxySettings proxy;\n\n @BeforeAll\n public static void setUp() {\n echoUrl = EchoServlet.Url;\n proxy = ProxyServlet.ProxySettings;\n }\n\n @ParameterizedTest @MethodSource(\"helloUrls\")\n void fetchViaProxy(String url) throws IOException {\n Connection con = Jsoup.connect(url)\n .proxy(proxy.hostname, proxy.port);\n\n Connection.Response res = con.execute();\n if (url.startsWith(\"http:/\")) assertVia(res); // HTTPS CONNECT won't have Via\n\n Document doc = res.parse();\n Element p = doc.expectFirst(\"p\");\n assertEquals(\"Hello, World!\", p.text());\n }\n\n private static Stream<String> helloUrls() {\n return Stream.of(HelloServlet.Url, HelloServlet.TlsUrl);\n }\n\n private static Stream<String> echoUrls() {\n return Stream.of(EchoServlet.Url, EchoServlet.TlsUrl);\n }\n\n private static void assertVia(Connection.Response res) {\n assertEquals(res.header(\"Via\"), ProxyServlet.Via);\n }\n\n @Test void redirectViaProxy() throws IOException {\n Connection.Response res = Jsoup\n .connect(RedirectServlet.Url)\n .data(RedirectServlet.LocationParam, echoUrl)\n .header(\"Random-Header-name\", \"hello\")\n .proxy(proxy.hostname, proxy.port)\n .execute();\n\n assertVia(res);\n Document doc = res.parse();\n assertEquals(echoUrl, doc.location());\n assertEquals(\"hello\", ihVal(\"Random-Header-name\", doc));\n assertVia(res);\n }\n\n @Test void proxyForSession() throws IOException {\n Connection session = Jsoup.newSession().proxy(proxy.hostname, proxy.port);\n\n Connection.Response medRes = session.newRequest(FileServlet.urlTo(\"/htmltests/medium.html\")).execute();\n Connection.Response largeRes = session.newRequest(FileServlet.urlTo(\"/htmltests/large.html\")).execute();\n\n assertVia(medRes);\n assertVia(largeRes);\n assertEquals(\"Medium HTML\", medRes.parse().title());\n assertEquals(\"Large HTML\", largeRes.parse().title());\n\n Connection.Response smedRes = session.newRequest(FileServlet.tlsUrlTo(\"/htmltests/medium.html\")).execute();\n Connection.Response slargeRes = session.newRequest(FileServlet.tlsUrlTo(\"/htmltests/large.html\")).execute();\n\n assertEquals(\"Medium HTML\", smedRes.parse().title());\n assertEquals(\"Large HTML\", slargeRes.parse().title());\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\")\n void canAuthenticateToProxy(String url) throws IOException {\n int closed = TestServer.closeAuthedProxyConnections(); // reset any existing authed connections from previous tests, so we can test the auth flow\n\n // the proxy wants auth, but not the server. HTTP and HTTPS, so tests direct proxy and CONNECT\n Connection session = Jsoup.newSession()\n .proxy(proxy.hostname, proxy.authedPort)\n .ignoreHttpErrors(true)\n .ignoreContentType(true); // ignore content type, as error served may not have a content type\n String password = AuthFilter.newProxyPassword();\n\n // fail first\n try {\n Connection.Response execute = session.newRequest(url)\n .execute();\n int code = execute.statusCode(); // no auth sent\n assertEquals(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED, code);\n } catch (IOException e) {\n assertAuthRequiredException(e);\n }\n\n try {\n AtomicInteger count = new AtomicInteger(0);\n Connection.Response res = session.newRequest(url)\n .auth(ctx -> {\n count.incrementAndGet();\n return ctx.credentials(AuthFilter.ProxyUser, password + \"wrong\"); // incorrect\n })\n .execute();\n assertEquals(MaxAttempts, count.get());\n assertEquals(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED, res.statusCode());\n } catch (IOException e) {\n assertAuthRequiredException(e);\n }\n\n AtomicInteger successCount = new AtomicInteger(0);\n Connection.Response successRes = session.newRequest(url)\n .auth(ctx -> {\n successCount.incrementAndGet();\n return ctx.credentials(AuthFilter.ProxyUser, password); // correct\n })\n .execute();\n assertEquals(1, successCount.get());\n assertEquals(HttpServletResponse.SC_OK, successRes.statusCode());\n }\n\n static void assertAuthRequiredException(IOException e) {\n // in CONNECT (for the HTTPS url), URLConnection will throw the proxy connect as a Stringly typed IO exception - \"Unable to tunnel through proxy. Proxy returns \"HTTP/1.1 407 Proxy Authentication Required\"\". (Not a response code)\n // Alternatively, some platforms (?) will report: \"No credentials provided\"\n String err = e.getMessage();\n if (!(err.contains(\"407\") || err.contains(\"No credentials provided\") || err.contains(\"exch.exchImpl\"))) {\n // https://github.com/jhy/jsoup/pull/2403 - Ubuntu Azul 25 throws `Cannot invoke \"jdk.internal.net.http.ExchangeImpl.cancel(java.io.IOException)\" because \"exch.exchImpl\" is null` here but is just from cancelling the 407 req\n System.err.println(\"Not a 407 exception? \" + e.getClass());\n e.printStackTrace(System.err);\n fail(\"Expected 407 Proxy Authentication Required, got: \" + err);\n }\n }\n\n @ParameterizedTest @MethodSource(\"echoUrls\")\n void canAuthToProxyAndServer(String url) throws IOException {\n String serverPassword = AuthFilter.newServerPassword();\n String proxyPassword = AuthFilter.newProxyPassword();\n AtomicInteger count = new AtomicInteger(0);\n\n Connection session = Jsoup.newSession() // both proxy and server will want auth\n .proxy(proxy.hostname, proxy.authedPort)\n .header(AuthFilter.WantsServerAuthentication, \"1\")\n .auth(auth -> {\n count.incrementAndGet();\n\n if (auth.isServer()) {\n assertEquals(url, auth.url().toString());\n assertEquals(AuthFilter.ServerRealm, auth.realm());\n return auth.credentials(AuthFilter.ServerUser, serverPassword);\n } else {\n assertTrue(auth.isProxy());\n return auth.credentials(AuthFilter.ProxyUser, proxyPassword);\n }\n });\n\n\n Connection.Response res = session.newRequest(url).execute();\n assertEquals(200, res.statusCode());\n assertEquals(2, count.get()); // hit server and proxy auth stages\n assertEquals(\"Webserver Environment Variables\", res.parse().title());\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/SafelistExtensionTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.TextUtil;\nimport org.jsoup.nodes.Attribute;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.safety.Safelist;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\n/**\n Check that we can extend Safelist methods\n */\npublic class SafelistExtensionTest {\n @Test public void canCustomizeSafeTests() {\n OpenSafelist openSafelist = new OpenSafelist(Safelist.relaxed());\n Safelist safelist = Safelist.relaxed();\n\n String html = \"<p><opentag openattr>Hello</opentag></p>\";\n\n String openClean = Jsoup.clean(html, openSafelist);\n String clean = Jsoup.clean(html, safelist);\n\n assertEquals(\"<p><opentag openattr=\\\"\\\">Hello</opentag></p>\", TextUtil.stripNewlines(openClean));\n assertEquals(\"<p>Hello</p>\", clean);\n }\n\n // passes tags and attributes starting with \"open\"\n private static class OpenSafelist extends Safelist {\n public OpenSafelist(Safelist safelist) {\n super(safelist);\n }\n\n @Override\n public boolean isSafeAttribute(String tagName, Element el, Attribute attr) {\n if (attr.getKey().startsWith(\"open\"))\n return true;\n return super.isSafeAttribute(tagName, el, attr);\n }\n\n @Override\n public boolean isSafeTag(String tag) {\n if (tag.startsWith(\"open\"))\n return true;\n return super.isSafeTag(tag);\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/SessionIT.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Connection;\nimport org.jsoup.Jsoup;\nimport org.jsoup.integration.servlets.FileServlet;\nimport org.jsoup.integration.servlets.SlowRider;\nimport org.jsoup.nodes.Document;\nimport org.junit.jupiter.api.BeforeAll;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.IOException;\nimport java.io.UncheckedIOException;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\n/** Integration tests to test longer running Connection */\npublic class SessionIT {\n @BeforeAll\n public static void setUp() {\n TestServer.start();\n }\n\n @Test\n public void multiThread() throws InterruptedException {\n int numThreads = 20;\n int numThreadLoops = 5;\n String[] urls = {\n FileServlet.urlTo(\"/htmltests/medium.html\"),\n FileServlet.urlTo(\"/htmltests/upload-form.html\"),\n FileServlet.urlTo(\"/htmltests/comments.html\"),\n FileServlet.urlTo(\"/htmltests/large.html\"),\n };\n String[] titles = {\n \"Medium HTML\",\n \"Upload Form Test\",\n \"A Certain Kind of Test\",\n \"Large HTML\"\n };\n ThreadCatcher catcher = new ThreadCatcher();\n\n Connection session = Jsoup.newSession();\n\n Thread[] threads = new Thread[numThreads];\n for (int threadNum = 0; threadNum < numThreads; threadNum++) {\n Thread thread = new Thread(() -> {\n for (int loop = 0; loop < numThreadLoops; loop++) {\n for (int i = 0; i < urls.length; i++) {\n try {\n Document doc = session.newRequest().url(urls[i]).get();\n assertEquals(titles[i], doc.title());\n } catch (IOException e) {\n throw new UncheckedIOException(e);\n }\n }\n }\n });\n thread.setName(\"Runner-\" + threadNum);\n thread.start();\n thread.setUncaughtExceptionHandler(catcher);\n threads[threadNum] = thread;\n }\n\n // now join them all\n for (Thread thread : threads) {\n thread.join();\n }\n\n assertEquals(0, catcher.exceptionCount.get());\n }\n\n // test that we throw a nice clear exception if you try to multi-thread by forget .newRequest()\n @Test\n public void multiThreadWithoutNewRequestBlowsUp() throws InterruptedException {\n int numThreads = 5;\n String url = SlowRider.Url + \"?\" + SlowRider.MaxTimeParam + \"=20000\"; // this makes sure that the first req is still executing whilst the others run\n String title = \"Slow Rider\";\n\n ThreadCatcher catcher = new ThreadCatcher();\n Connection session = Jsoup.newSession();\n\n // run first slow request\n AtomicInteger successful = new AtomicInteger();\n Thread slow = new Thread(() -> {\n try {\n Document doc = session.url(url).get();\n assertNotNull(doc);\n } catch (IOException e) {\n if (!isInterruptedException(e))\n throw new UncheckedIOException(e);\n }\n });\n slow.start();\n Thread.sleep(100); // yield so that thread can start before the next do\n\n // spawn others, should fail\n Thread[] threads = new Thread[numThreads];\n for (int threadNum = 0; threadNum < numThreads; threadNum++) {\n Thread thread = new Thread(() -> {\n try {\n Document doc = session.url(url).get();\n assertEquals(title, doc.title());\n successful.getAndIncrement();\n } catch (IOException e) {\n throw new UncheckedIOException(e);\n }\n });\n thread.setName(\"Runner-\" + threadNum);\n thread.setUncaughtExceptionHandler(catcher);\n thread.start();\n threads[threadNum] = thread;\n }\n\n // now join them all\n for (Thread thread : threads) {\n thread.join();\n }\n // cancel the slow runner so we can wrap up the test quicker\n slow.interrupt();\n\n // only one should have passed, rest should have blown up (assuming the started whilst other was running)\n //assertEquals(numThreads - 1, catcher.multiThreadExceptions.get());\n //assertEquals(numThreads - 1, catcher.exceptionCount.get());\n\n /* The checks above work when all 20 threads are executed within 10 seconds. However, depending on how cloudy it\n is when the CI jobs are run, they may not all complete in time. As of writing that appears most commonly on the\n macOS runners, which appear overtaxed. That makes this test flaky. So we relax the test conditions, and make\n sure at least just one passed and one failed. That's OK in prod as well, because we are only concerned about\n concurrent execution, which the impl does detect correctly. */\n assertEquals(0, successful.get());\n assertTrue(catcher.multiThreadExceptions.get() > 0);\n assertEquals(catcher.multiThreadExceptions.get(), catcher.exceptionCount.get()); // all exceptions are multi-threaded\n }\n\n @Test\n public void multiThreadWithProgressListener() throws InterruptedException {\n // tests that we can use one progress listener for multiple URLs and threads.\n int numThreads = 10;\n String[] urls = {\n FileServlet.urlTo(\"/htmltests/medium.html\"),\n FileServlet.urlTo(\"/htmltests/upload-form.html\"),\n FileServlet.urlTo(\"/htmltests/comments.html\"),\n FileServlet.urlTo(\"/htmltests/large.html\"),\n };\n Set<String> seenUrls = ConcurrentHashMap.newKeySet();\n AtomicInteger completedCount = new AtomicInteger(0);\n ThreadCatcher catcher = new ThreadCatcher();\n\n Connection session = Jsoup.newSession()\n .onResponseProgress((processed, total, percent, response) -> {\n if (percent == 100.0f) {\n //System.out.println(\"Completed \" + Thread.currentThread().getName() + \"- \" + response.url());\n seenUrls.add(response.url().toExternalForm());\n completedCount.incrementAndGet();\n }\n });\n\n Thread[] threads = new Thread[numThreads];\n for (int threadNum = 0; threadNum < numThreads; threadNum++) {\n Thread thread = new Thread(() -> {\n for (String url : urls) {\n try {\n Connection con = session.newRequest().url(url);\n con.get();\n } catch (IOException e) {\n throw new UncheckedIOException(e);\n }\n }\n });\n thread.setName(\"Runner-\" + threadNum);\n thread.setUncaughtExceptionHandler(catcher);\n thread.start();\n threads[threadNum] = thread;\n }\n\n // now join them all\n for (Thread thread : threads) {\n thread.join();\n }\n\n assertEquals(0, catcher.exceptionCount.get());\n assertEquals(urls.length, seenUrls.size());\n assertEquals(urls.length * numThreads, completedCount.get());\n }\n\n\n static class ThreadCatcher implements Thread.UncaughtExceptionHandler {\n AtomicInteger exceptionCount = new AtomicInteger();\n AtomicInteger multiThreadExceptions = new AtomicInteger();\n\n @Override\n public void uncaughtException(Thread t, Throwable e) {\n if (e.getMessage() != null && e.getMessage().contains(\"Multiple threads\")) {\n multiThreadExceptions.incrementAndGet();\n } else if (!isInterruptedException(e)) {\n e.printStackTrace();\n }\n exceptionCount.incrementAndGet();\n }\n }\n\n private static boolean isInterruptedException(Throwable e) {\n Throwable cause = e;\n while (cause != null) {\n if (cause instanceof InterruptedException) return true;\n cause = cause.getCause();\n }\n return false;\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/SessionTest.java",
"content": "package org.jsoup.integration;\n\nimport org.jsoup.Connection;\nimport org.jsoup.Jsoup;\nimport org.jsoup.integration.servlets.CookieServlet;\nimport org.jsoup.integration.servlets.EchoServlet;\nimport org.jsoup.integration.servlets.FileServlet;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.parser.Parser;\nimport org.jsoup.parser.Tag;\nimport org.jsoup.parser.TagSet;\nimport org.jsoup.select.Elements;\nimport org.junit.jupiter.api.BeforeAll;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.util.Map;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\n\npublic class SessionTest {\n @BeforeAll\n public static void setUp() {\n TestServer.start();\n }\n\n private static Elements keyEls(String key, Document doc) {\n return doc.select(\"th:contains(\" + key + \") + td\");\n }\n\n private static String keyText(String key, Document doc) {\n return doc.selectFirst(\"th:contains(\" + key + \") + td\").text();\n }\n\n @Test\n public void testPathScopedCookies() throws IOException {\n final Connection session = Jsoup.newSession();\n final String userAgent = \"Jsoup Testalot v0.1\";\n\n session.userAgent(userAgent);\n session.url(CookieServlet.Url);\n\n // should have no cookies:\n Connection con1 = session.newRequest();\n Document doc1 = con1.get();\n assertEquals(0, doc1.select(\"table tr\").size()); // none sent to servlet\n\n // set the cookies\n Connection con2 = session.newRequest().data(CookieServlet.SetCookiesParam, \"1\");\n Document doc2 = con2.get();\n assertEquals(0, doc2.select(\"table tr\").size()); // none sent to servlet - we just got them!\n Map<String, String> cookies = con2.response().cookies(); // simple cookie response, all named \"One\", so should be last sent\n assertEquals(2, cookies.size());\n assertEquals(\"EchoServlet\", cookies.get(\"One\"));\n\n // test that all response cookies are present, even if would not be sent for this request path (i.e. cookies() res can be set on a req, without using sessions)\n assertEquals(\"Override\", cookies.get(\"Two\"));\n\n // todo - interrogate cookie-store\n\n // check that they are sent and filtered to the right path\n Connection con3 = session.newRequest();\n Document doc3 = con3.get();\n assertCookieServlet(doc3);\n\n Document echo = session.newRequest().url(EchoServlet.Url).get();\n assertEchoServlet(echo);\n assertEquals(userAgent, keyText(\"User-Agent\", echo)); // check that customer user agent sent on session arrived\n\n // check that cookies aren't set out of the session\n Document doc4 = Jsoup.newSession().url(CookieServlet.Url).get();\n assertEquals(0, doc4.select(\"table tr\").size()); // none sent to servlet\n\n // check can add local ones also\n Document doc5 = session.newRequest().cookie(\"Bar\", \"Qux\").get();\n Elements doc5Bar = keyEls(\"Bar\", doc5);\n assertEquals(\"Qux\", doc5Bar.first().text());\n }\n\n // validate that only cookies set by cookie servlet get to the cookie servlet path\n private void assertCookieServlet(Document doc) {\n assertEquals(2, doc.select(\"table tr\").size()); // two of three sent to servlet (/ and /CookieServlet)\n Elements doc3Els = keyEls(\"One\", doc);\n assertEquals(2, doc3Els.size());\n assertEquals(\"CookieServlet\", doc3Els.get(0).text()); // ordered by most specific path\n assertEquals(\"Root\", doc3Els.get(1).text()); // ordered by most specific path\n }\n\n // validate that only for echo servlet\n private void assertEchoServlet(Document doc) {\n Elements echoEls = keyEls(\"Cookie: One\", doc); // two of three sent to servlet (/ and /EchoServlet)\n assertEquals(2, echoEls.size());\n assertEquals(\"EchoServlet\", echoEls.get(0).text()); // ordered by most specific path - /Echo\n assertEquals(\"Root\", echoEls.get(1).text()); // ordered by most specific path - /\n }\n\n @Test\n public void testPathScopedCookiesOnRedirect() throws IOException {\n Connection session = Jsoup.newSession();\n\n Document doc1 = session.newRequest()\n .url(CookieServlet.Url)\n .data(CookieServlet.LocationParam, EchoServlet.Url)\n .data(CookieServlet.SetCookiesParam, \"1\")\n .get();\n\n // we should be redirected to the echo servlet with cookies\n assertEquals(EchoServlet.Url, doc1.location());\n assertEchoServlet(doc1); // checks we only have /echo cookies\n\n Document doc2 = session.newRequest()\n .url(EchoServlet.Url)\n .get();\n assertEchoServlet(doc2); // test retained in session\n\n Document doc3 = session.newRequest()\n .url(CookieServlet.Url)\n .get();\n assertCookieServlet(doc3); // and so were the /cookie cookies\n }\n\n @Test\n public void testCanChangeParsers() throws IOException {\n Connection session = Jsoup.newSession().parser(Parser.xmlParser());\n\n String xmlUrl = FileServlet.urlTo(\"/htmltests/xml-test.xml\");\n String xmlVal = \"<doc><val>One<val>Two</val>Three</val></doc>\\n\";\n\n Document doc1 = session.newRequest().url(xmlUrl).get();\n assertEquals(xmlVal, doc1.html()); // not HTML normed, used XML parser\n\n Document doc2 = session.newRequest().parser(Parser.htmlParser()).url(xmlUrl).get();\n assertTrue(doc2.html().startsWith(\"<html>\"));\n\n Document doc3 = session.newRequest().url(xmlUrl).get();\n assertEquals(xmlVal, doc3.html()); // did not blow away xml default\n }\n\n @Test\n public void sessionTagSetDoesNotMutateRoot() {\n Connection session = Jsoup.newSession();\n TagSet rootTags = session.request().parser().tagSet();\n\n int rootNamespacesBefore = tagSetNamespaceCount(rootTags);\n\n Connection request = session.newRequest();\n Parser parser = request.request().parser();\n parser.parseInput(\"<custom>One <b>Two</b></custom>\", \"http://example.com/\");\n\n int rootNamespacesAfter = tagSetNamespaceCount(rootTags);\n assertEquals(rootNamespacesBefore, rootNamespacesAfter);\n }\n\n @Test\n public void sessionTagSetCustomizerDoesNotMutateRoot() {\n Connection session = Jsoup.newSession();\n TagSet rootTags = session.request().parser().tagSet();\n rootTags.onNewTag(tag -> {\n if (!tag.isKnownTag())\n tag.set(Tag.RcData);\n });\n\n int rootNamespacesBefore = tagSetNamespaceCount(rootTags);\n\n Connection request = session.newRequest();\n Parser parser = request.request().parser();\n Document doc = parser.parseInput(\"<custom>One <b>Two</b></custom>\", \"https://example.com/\");\n assertEquals(0, doc.select(\"custom b\").size());\n\n int rootNamespacesAfter = tagSetNamespaceCount(rootTags);\n assertEquals(rootNamespacesBefore, rootNamespacesAfter);\n }\n\n private static int tagSetNamespaceCount(TagSet tagSet) {\n try {\n Field tagsField = TagSet.class.getDeclaredField(\"tags\");\n tagsField.setAccessible(true);\n Map<?, ?> tags = (Map<?, ?>) tagsField.get(tagSet);\n return tags.size();\n } catch (ReflectiveOperationException e) {\n throw new RuntimeException(e);\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/TestServer.java",
"content": "package org.jsoup.integration;\n\nimport org.eclipse.jetty.http.HttpVersion;\nimport org.eclipse.jetty.server.Connector;\nimport org.eclipse.jetty.server.HttpConfiguration;\nimport org.eclipse.jetty.server.HttpConnectionFactory;\nimport org.eclipse.jetty.server.SecureRequestCustomizer;\nimport org.eclipse.jetty.server.Server;\nimport org.eclipse.jetty.server.ServerConnector;\nimport org.eclipse.jetty.server.SslConnectionFactory;\nimport org.eclipse.jetty.server.handler.HandlerWrapper;\nimport org.eclipse.jetty.servlet.FilterHolder;\nimport org.eclipse.jetty.servlet.FilterMapping;\nimport org.eclipse.jetty.servlet.ServletHandler;\nimport org.eclipse.jetty.util.log.Log;\nimport org.eclipse.jetty.util.log.StdErrLog;\nimport org.eclipse.jetty.util.ssl.SslContextFactory;\nimport org.jsoup.integration.servlets.AuthFilter;\nimport org.jsoup.integration.servlets.BaseServlet;\nimport org.jsoup.integration.servlets.ProxyServlet;\n\nimport javax.net.ssl.HttpsURLConnection;\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.SSLSocketFactory;\nimport javax.net.ssl.TrustManager;\nimport javax.net.ssl.TrustManagerFactory;\nimport java.io.File;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.net.InetSocketAddress;\nimport java.nio.file.Files;\nimport java.security.KeyManagementException;\nimport java.security.KeyStore;\nimport java.security.KeyStoreException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.cert.CertificateException;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class TestServer {\n static int Port;\n static int TlsPort;\n\n private static final String Localhost = \"localhost\";\n private static final String KeystorePassword = \"hunter2\";\n\n private static final Server Jetty = newServer();\n private static final ServletHandler JettyHandler = new ServletHandler();\n private static final Server Proxy = newServer();\n private static final Server AuthedProxy = newServer();\n private static final HandlerWrapper ProxyHandler = new HandlerWrapper();\n private static final HandlerWrapper AuthedProxyHandler = new HandlerWrapper();\n private static final ProxySettings ProxySettings = new ProxySettings();\n\n private static Server newServer() {\n // logs to stdout, so not highlighted as errors in Maven test runs\n StdErrLog logger = new StdErrLog();\n logger.setStdErrStream(System.out);\n Log.setLog(logger);\n\n return new Server(new InetSocketAddress(Localhost, 0));\n }\n\n static {\n Jetty.setHandler(JettyHandler);\n Proxy.setHandler(ProxyHandler);\n AuthedProxy.setHandler(AuthedProxyHandler);\n\n // TLS setup:\n try {\n File keystoreFile = ParseTest.getFile(\"/local-cert/server.pfx\");\n if (!keystoreFile.exists()) throw new FileNotFoundException(keystoreFile.toString());\n addHttpsConnector(keystoreFile, Jetty);\n setupDefaultTrust(keystoreFile);\n } catch (Exception e) {\n throw new IllegalStateException(e);\n }\n }\n\n private TestServer() {\n }\n\n public static void start() {\n synchronized (Jetty) {\n if (Jetty.isStarted()) return;\n\n try {\n Jetty.start();\n JettyHandler.addFilterWithMapping(new FilterHolder(new AuthFilter(false, false)), \"/*\", FilterMapping.ALL);\n Connector[] jcons = Jetty.getConnectors();\n Port = ((ServerConnector) jcons[0]).getLocalPort();\n TlsPort = ((ServerConnector) jcons[1]).getLocalPort();\n\n ProxyHandler.setHandler(ProxyServlet.createHandler(false)); // includes proxy, CONNECT proxy, and Auth filters\n Proxy.start();\n ProxySettings.port = ((ServerConnector) Proxy.getConnectors()[0]).getLocalPort();\n\n AuthedProxyHandler.setHandler(ProxyServlet.createHandler(true));\n AuthedProxy.start();\n ProxySettings.authedPort = ((ServerConnector) AuthedProxy.getConnectors()[0]).getLocalPort();\n } catch (Exception e) {\n throw new IllegalStateException(e);\n }\n }\n }\n\n /**\n Close any current connections to the authed proxy. Tunneled connections only authenticate in their first\n CONNECT, and may be kept alive and reused. So when we want to test unauthed - authed flows, we need to disconnect\n them first.\n */\n static int closeAuthedProxyConnections() {\n ServerConnector connector = (ServerConnector) AuthedProxy.getConnectors()[0];\n AtomicInteger count = new AtomicInteger();\n connector.getConnectedEndPoints().forEach(endPoint -> {\n endPoint.close();\n count.getAndIncrement();\n });\n return count.get();\n }\n\n public static ServletUrls map(Class<? extends BaseServlet> servletClass) {\n synchronized (Jetty) {\n if (!Jetty.isStarted())\n start(); // if running out of the test cases\n\n String path = \"/\" + servletClass.getSimpleName();\n JettyHandler.addServletWithMapping(servletClass, path + \"/*\");\n String url = \"http://\" + Localhost + \":\" + Port + path;\n String tlsUrl = \"https://\" + Localhost + \":\" + TlsPort + path;\n\n return new ServletUrls(url, tlsUrl);\n }\n }\n\n public static class ServletUrls {\n public final String url;\n public final String tlsUrl;\n\n public ServletUrls(String url, String tlsUrl) {\n this.url = url;\n this.tlsUrl = tlsUrl;\n }\n }\n\n public static ProxySettings proxySettings() {\n synchronized (Jetty) {\n if (!Jetty.isStarted())\n start();\n\n return ProxySettings;\n }\n }\n\n //public static String proxy\n public static class ProxySettings {\n final String hostname = Localhost;\n int port;\n int authedPort;\n }\n\n private static void addHttpsConnector(File keystoreFile, Server server) {\n // Cribbed from https://github.com/jetty/jetty.project/blob/jetty-9.4.x/examples/embedded/src/main/java/org/eclipse/jetty/embedded/LikeJettyXml.java\n SslContextFactory sslContextFactory = new SslContextFactory.Server();\n String path = keystoreFile.getAbsolutePath();\n sslContextFactory.setKeyStorePath(path);\n sslContextFactory.setKeyStorePassword(KeystorePassword);\n sslContextFactory.setKeyManagerPassword(KeystorePassword);\n sslContextFactory.setTrustStorePath(path);\n sslContextFactory.setTrustStorePassword(KeystorePassword);\n\n HttpConfiguration httpConfig = new HttpConfiguration();\n httpConfig.setSecureScheme(\"https\");\n HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);\n httpsConfig.addCustomizer(new SecureRequestCustomizer());\n\n ServerConnector sslConnector = new ServerConnector(\n server,\n new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),\n new HttpConnectionFactory(httpsConfig));\n sslConnector.setHost(Localhost);\n server.addConnector(sslConnector);\n }\n\n private static void setupDefaultTrust(File keystoreFile) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException {\n // Configure HttpsUrlConnection (jsoup) to trust (only) this cert\n KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());\n trustStore.load(Files.newInputStream(keystoreFile.toPath()), KeystorePassword.toCharArray());\n TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());\n trustManagerFactory.init(trustStore);\n TrustManager[] managers = trustManagerFactory.getTrustManagers();\n SSLContext tls = SSLContext.getInstance(\"TLS\");\n tls.init(null, managers, null);\n SSLContext.setDefault(tls);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/AuthFilter.java",
"content": "package org.jsoup.integration.servlets;\n\nimport javax.servlet.Filter;\nimport javax.servlet.FilterChain;\nimport javax.servlet.FilterConfig;\nimport javax.servlet.ServletException;\nimport javax.servlet.ServletRequest;\nimport javax.servlet.ServletResponse;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Base64;\n\n/**\n A filter to test basic authenticated requests. If the request header \"X-Wants-Authentication\" is set, or if\n alwaysWantsAuth is enabled, the filter is invoked, and requests must send the correct user authentication details.\n */\npublic class AuthFilter implements Filter {\n public static final String WantsServerAuthentication = \"X-Wants-ServerAuthentication\";\n public static final String ServerUser = \"admin\";\n public static final String ServerRealm = \"jsoup test server authentication realm\";\n private static volatile String ServerPassword = newServerPassword();\n\n public static final String WantsProxyAuthentication = \"X-Wants-ProxyAuthentication\";\n public static final String ProxyUser = \"foxyproxy\";\n public static final String ProxyRealm = \"jsoup test proxy authentication realm\";\n private static volatile String ProxyPassword = newProxyPassword();\n\n private final boolean alwaysWantsAuth; // we run a particular port that always wants auth - so the CONNECT tunnels can be authed. (The Java proxy tunnel CONNECT request strips the wants-auth headers)\n private final boolean forProxy;\n private final String wantsHeader;\n private final String authorizationHeader;\n\n /**\n Creates an Authentication Filter with hardcoded credential expectations.\n * @param alwaysWantsAuth true if this filter should always check for authentication, regardless of the Wants Auth header\n * @param forProxy true if this wraps a Proxy and should use Proxy-Authenticate headers, credentials etc. False\n * if wrapping the web server.\n */\n public AuthFilter(boolean alwaysWantsAuth, boolean forProxy) {\n this.alwaysWantsAuth = alwaysWantsAuth;\n this.forProxy = forProxy;\n\n wantsHeader = forProxy ? WantsProxyAuthentication : WantsServerAuthentication;\n authorizationHeader = forProxy ? \"Proxy-Authorization\" : \"Authorization\";\n }\n\n private static String newPassword() {\n return \"pass-\" + Math.random();\n }\n\n // passwords get rotated in tests so that Java's auth cache is invalidated and a new auth callback occurs.\n // requires tests hitting these are called serially.\n public static String newServerPassword() {\n return ServerPassword = newPassword() + \"-server\";\n }\n\n public static String newProxyPassword() {\n return ProxyPassword = newPassword() + \"-proxy\";\n }\n\n @Override public void init(FilterConfig filterConfig) throws ServletException {}\n\n @Override\n public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {\n HttpServletRequest req = (HttpServletRequest) request;\n HttpServletResponse res = (HttpServletResponse) response;\n\n boolean accessGranted = checkAuth(req);\n if (accessGranted) {\n chain.doFilter(request, response);\n return;\n }\n\n // Wants but failed auth - send appropriate header:\n if (forProxy) {\n res.setHeader(\"Proxy-Authenticate\", \"Basic realm=\\\"\" + ProxyRealm + \"\\\"\");\n // ^^ Duped in ProxyServlet for CONNECT\n res.sendError(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED);\n } else {\n res.setHeader(\"WWW-Authenticate\", \"Basic realm=\\\"\" + ServerRealm + \"\\\"\");\n res.sendError(HttpServletResponse.SC_UNAUTHORIZED);\n }\n }\n\n @Override public void destroy() {}\n\n public boolean checkAuth(HttpServletRequest req) {\n if (alwaysWantsAuth || req.getHeader(wantsHeader) != null) {\n String authHeader = req.getHeader(authorizationHeader);\n if (authHeader != null) {\n int space = authHeader.indexOf(' ');\n if (space > 0) {\n String value = authHeader.substring(space + 1);\n String expected = forProxy ?\n (ProxyUser + \":\" + ProxyPassword) :\n (ServerUser + \":\" + ServerPassword);\n String base64 = Base64.getEncoder().encodeToString(expected.getBytes(StandardCharsets.UTF_8));\n return base64.equals(value); // if passed auth\n }\n }\n return false; // unexpected header value\n }\n return true; // auth not required\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/BaseServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport javax.servlet.ServletException;\nimport javax.servlet.http.HttpServlet;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\n\npublic abstract class BaseServlet extends HttpServlet {\n static final String TextHtml = \"text/html; charset=UTF-8\";\n\n abstract protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException;\n\n @Override\n protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {\n doIt(req, res);\n }\n\n @Override\n protected void doPost(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {\n doIt(req, res);\n }\n\n @Override\n protected void doPut(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {\n doIt(req, res);\n }\n\n @Override\n protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {\n doIt(req, res);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/CookieServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.Cookie;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\nimport java.io.PrintWriter;\n\npublic class CookieServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(CookieServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n public static final String SetCookiesParam = \"setCookies\";\n public static final String LocationParam = \"loc\";\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n // Do we want to set cookies?\n if (req.getParameter(SetCookiesParam) != null)\n setCookies(res);\n\n // Do we want to redirect elsewhere?\n String loc = req.getParameter(LocationParam);\n if (loc != null) {\n res.sendRedirect(loc);\n return;\n }\n\n // print out the cookies that were received\n res.setContentType(TextHtml);\n res.setStatus(200);\n\n PrintWriter w = res.getWriter();\n w.println(\"<table>\");\n final Cookie[] cookies = req.getCookies();\n if (cookies != null) {\n for (Cookie cookie : cookies) {\n EchoServlet.write(w, cookie.getName(), cookie.getValue());\n }\n }\n w.println(\"</table>\");\n }\n\n private void setCookies(HttpServletResponse res) {\n Cookie one = new Cookie(\"One\", \"Root\");\n one.setPath(\"/\");\n res.addCookie(one);\n\n Cookie two = new Cookie(\"One\", \"CookieServlet\");\n two.setPath(\"/CookieServlet\");\n two.setHttpOnly(true);\n two.setComment(\"Quite nice\");\n res.addCookie(two);\n\n Cookie three = new Cookie(\"One\", \"EchoServlet\");\n three.setPath(\"/EchoServlet\");\n res.addCookie(three);\n\n Cookie four = new Cookie(\"Two\", \"NoSuchPath\");\n four.setPath(\"/bogus\");\n res.addCookie(four);\n\n Cookie five = new Cookie(\"Two\", \"Override\");\n five.setPath(\"/bogus\");\n res.addCookie(five);\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/DeflateServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.zip.Deflater;\nimport java.util.zip.DeflaterOutputStream;\n\npublic class DeflateServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(DeflateServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n res.setContentType(TextHtml);\n res.setStatus(HttpServletResponse.SC_OK);\n res.setHeader(\"Content-Encoding\", \"deflate\");\n\n String doc = \"<p>Hello, World!<p>That should be enough, right?<p>Hello, World!<p>That should be enough, right?\";\n\n DeflaterOutputStream stream = new DeflaterOutputStream(\n res.getOutputStream(),\n new Deflater(Deflater.BEST_COMPRESSION, true)); // true = nowrap zlib headers\n\n stream.write(doc.getBytes(StandardCharsets.UTF_8));\n stream.close();\n }\n\n // allow the servlet to run as a main program, for local test\n public static void main(String[] args) {\n TestServer.start();\n System.out.println(Url);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/EchoServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.eclipse.jetty.server.Request;\nimport org.jsoup.helper.DataUtil;\nimport org.jsoup.internal.StringUtil;\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.MultipartConfigElement;\nimport javax.servlet.ServletException;\nimport javax.servlet.http.Cookie;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport javax.servlet.http.Part;\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collection;\nimport java.util.Enumeration;\n\nimport static org.jsoup.nodes.Entities.escape;\n\npublic class EchoServlet extends BaseServlet {\n public static final String CodeParam = \"code\";\n private static final int DefaultCode = HttpServletResponse.SC_OK;\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(EchoServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {\n int intCode = DefaultCode;\n String code = req.getHeader(CodeParam);\n if (code != null)\n intCode = Integer.parseInt(code);\n\n boolean isMulti = maybeEnableMultipart(req);\n\n res.setContentType(TextHtml);\n res.setStatus(intCode);\n // no-cache headers for test\n res.addHeader(\"Cache-Control\", \"no-cache\");\n res.addHeader(\"Cache-Control\", \"no-store\");\n\n PrintWriter w = res.getWriter();\n\n w.write(\"<title>Webserver Environment Variables\\n\" +\n \" \\n\" +\n \" \\n\" +\n \" \");\n\n // some get items\n write(w, \"Method\", req.getMethod());\n write(w, \"Request URI\", req.getRequestURI());\n write(w, \"Path Info\", req.getPathInfo());\n write(w, \"Query String\", req.getQueryString());\n\n // request headers (why is it an enumeration?)\n Enumeration headerNames = req.getHeaderNames();\n while (headerNames.hasMoreElements()) {\n String header = headerNames.nextElement();\n Enumeration headers = req.getHeaders(header);\n while (headers.hasMoreElements()) {\n write(w, header, headers.nextElement());\n }\n }\n\n // cookies\n final Cookie[] cookies = req.getCookies();\n if (cookies != null) {\n for (Cookie cookie : cookies) {\n EchoServlet.write(w, \"Cookie: \" + cookie.getName(), cookie.getValue());\n }\n }\n\n // the request params\n Enumeration parameterNames = req.getParameterNames();\n while (parameterNames.hasMoreElements()) {\n String name = parameterNames.nextElement();\n String[] values = req.getParameterValues(name);\n write(w, name, StringUtil.join(values, \", \"));\n }\n\n // post body\n ByteBuffer byteBuffer = DataUtil.readToByteBuffer(req.getInputStream(), 0);\n String postData = new String(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit(), StandardCharsets.UTF_8);\n if (!StringUtil.isBlank(postData)) {\n write(w, \"Post Data\", postData);\n }\n\n // file uploads\n if (isMulti) {\n Collection parts = req.getParts();\n write(w, \"Parts\", String.valueOf(parts.size()));\n\n for (Part part : parts) {\n String name = part.getName();\n write(w, \"Part \" + name + \" ContentType\", part.getContentType());\n write(w, \"Part \" + name + \" Name\", name);\n write(w, \"Part \" + name + \" Filename\", part.getSubmittedFileName());\n write(w, \"Part \" + name + \" Size\", String.valueOf(part.getSize()));\n part.delete();\n }\n }\n\n w.println(\"
\");\n }\n\n static void write(PrintWriter w, String key, String val) {\n w.println(\"| \" + escape(key) + \" | \" + escape(val) + \" |
|---|
\");\n }\n\n // allow the servlet to run as a main program, for local test\n public static void main(String[] args) {\n TestServer.start();\n System.out.println(\"Listening on \" + Url + \" and \" + TlsUrl);\n }\n\n private static boolean maybeEnableMultipart(HttpServletRequest req) {\n boolean isMulti = req.getContentType() != null\n && req.getContentType().startsWith(\"multipart/form-data\");\n\n if (isMulti) {\n req.setAttribute(Request.MULTIPART_CONFIG_ELEMENT, new MultipartConfigElement(\n System.getProperty(\"java.io.tmpdir\")));\n }\n return isMulti;\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/FileServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.ParseTest;\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.ServletOutputStream;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Files;\n\npublic class FileServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(FileServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n public static final String ContentTypeParam = \"contentType\";\n public static final String HtmlType = \"text/html\";\n static final String XmlType = \"text/xml\";\n public static final String SuppressContentLength = \"surpriseMe\";\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n String contentType = req.getParameter(ContentTypeParam);\n if (contentType == null) {\n contentType = HtmlType;\n if (req.getPathInfo().contains(\".xml\")) contentType = XmlType;\n }\n String location = req.getPathInfo();\n\n File file = ParseTest.getFile(location);\n if (file.exists()) {\n res.setContentType(contentType);\n if (file.getName().endsWith(\"gz\"))\n res.addHeader(\"Content-Encoding\", \"gzip\");\n if (req.getParameter(SuppressContentLength) == null)\n res.setContentLength((int) file.length());\n res.setStatus(HttpServletResponse.SC_OK);\n\n ServletOutputStream out = res.getOutputStream();\n Files.copy(file.toPath(), out);\n out.flush();\n } else {\n res.sendError(HttpServletResponse.SC_NOT_FOUND);\n }\n }\n\n public static String urlTo(String path) {\n return Url + path;\n }\n\n public static String tlsUrlTo(String path) {\n return TlsUrl + path;\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/HelloServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\n\npublic class HelloServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(HelloServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n res.setContentType(TextHtml);\n res.setStatus(HttpServletResponse.SC_OK);\n\n String doc = \"Hello, World!\";\n res.getWriter().write(doc);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/InterruptedServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\nimport org.jsoup.parser.CharacterReaderTest;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\n\npublic class InterruptedServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(InterruptedServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n public static final String Magnitude = \"magnitude\";\n public static final String Larger = \"larger\";\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n String magnitude = req.getParameter(Magnitude);\n magnitude = magnitude == null ? \"\" : magnitude;\n res.setContentType(TextHtml);\n res.setStatus(HttpServletResponse.SC_OK);\n\n StringBuilder sb = new StringBuilder();\n sb.append(\"
Something\");\n while (sb.length() <= 32 * 1024) {\n sb.append(\"A suitable amount of data.
\\n\");\n }\n sb.append(\"Finale.
\");\n String data = sb.toString();\n\n int contentLength = magnitude.equals(Larger) ? data.length() * 2 : data.length() / 2;\n res.setContentLength(contentLength);\n\n res.getWriter().write(data);\n\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/ProxyServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.eclipse.jetty.client.api.Response;\nimport org.eclipse.jetty.proxy.AsyncProxyServlet;\nimport org.eclipse.jetty.proxy.ConnectHandler;\nimport org.eclipse.jetty.server.Handler;\nimport org.eclipse.jetty.servlet.FilterHolder;\nimport org.eclipse.jetty.servlet.FilterMapping;\nimport org.eclipse.jetty.servlet.ServletHandler;\nimport org.eclipse.jetty.servlet.ServletHolder;\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\n\nimport static org.jsoup.integration.servlets.AuthFilter.ProxyRealm;\n\npublic class ProxyServlet extends AsyncProxyServlet {\n public static TestServer.ProxySettings ProxySettings = TestServer.proxySettings();\n public static String Via = \"1.1 jsoup test proxy\";\n\n static {\n System.setProperty(\"jdk.http.auth.tunneling.disabledSchemes\", \"\");\n // removes Basic, which is otherwise excluded from auth for CONNECT tunnels\n }\n\n public static Handler createHandler(boolean alwaysAuth) {\n // ConnectHandler wraps this ProxyServlet and handles CONNECT, which sets up a tunnel for HTTPS requests and is\n // opaque to the proxy. The ProxyServlet handles simple HTTP requests.\n AuthFilter authFilter = new AuthFilter(alwaysAuth, true);\n ConnectHandler connectHandler = new ConnectProxy(authFilter);\n ServletHandler proxyHandler = new ServletHandler();\n proxyHandler.addFilterWithMapping(new FilterHolder(authFilter), \"/*\", FilterMapping.ALL); // auth for HTTP proxy\n ServletHolder proxyServletHolder = new ServletHolder(ProxyServlet.class); // Holder wraps as it requires maxThreads initialization\n proxyServletHolder.setAsyncSupported(true);\n proxyServletHolder.setInitParameter(\"maxThreads\", \"200\");\n proxyHandler.addServletWithMapping(proxyServletHolder, \"/*\");\n connectHandler.setHandler(proxyHandler);\n\n return connectHandler;\n }\n\n @Override\n protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) {\n super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse);\n proxyResponse.addHeader(\"Via\", Via);\n }\n\n /** Supports CONNECT tunnels */\n static class ConnectProxy extends ConnectHandler {\n final AuthFilter authFilter;\n\n public ConnectProxy(AuthFilter authFilter) {\n this.authFilter = authFilter;\n }\n\n @Override\n protected boolean handleAuthentication(HttpServletRequest req, HttpServletResponse res, String address) {\n boolean accessGranted = authFilter.checkAuth(req);\n //System.err.println(\"CONNECT AUTH: \" + accessGranted);\n\n // need to add the desired auth header if not granted. Returning false here will also send 407 header\n if (!accessGranted) {\n res.setHeader(\"Proxy-Authenticate\", \"Basic realm=\\\"\" + ProxyRealm + \"\\\"\");\n }\n return accessGranted;\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/RedirectServlet.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.Cookie;\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\n\npublic class RedirectServlet extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(RedirectServlet.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n public static final String LocationParam = \"loc\";\n public static final String CodeParam = \"code\";\n public static final String SetCookiesParam = \"setCookies\";\n private static final int DefaultCode = HttpServletResponse.SC_MOVED_TEMPORARILY;\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n String location = req.getParameter(LocationParam);\n if (location == null)\n location = \"\";\n\n int intCode = DefaultCode;\n String code = req.getParameter(CodeParam);\n if (code != null)\n intCode = Integer.parseInt(code);\n\n if (req.getParameter(SetCookiesParam) != null) {\n res.addCookie(new Cookie(\"token\", \"asdfg123\"));\n res.addCookie(new Cookie(\"uid\", \"foobar\"));\n res.addCookie(new Cookie(\"uid\", \"jhy\")); // dupe, should use latter\n }\n\n res.setHeader(\"Location\", location);\n res.setStatus(intCode);\n res.flushBuffer();\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/integration/servlets/SlowRider.java",
"content": "package org.jsoup.integration.servlets;\n\nimport org.jsoup.integration.TestServer;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.IOException;\nimport java.io.PrintWriter;\n\n/**\n * Slowly, interminably writes output. For the purposes of testing timeouts and interrupts.\n */\npublic class SlowRider extends BaseServlet {\n public static final String Url;\n public static final String TlsUrl;\n static {\n TestServer.ServletUrls urls = TestServer.map(SlowRider.class);\n Url = urls.url;\n TlsUrl = urls.tlsUrl;\n }\n private static final int SleepTime = 2000;\n public static final String MaxTimeParam = \"maxTime\";\n public static final String IntroSizeParam = \"introSize\";\n\n @Override\n protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {\n pause(1000);\n res.setContentType(TextHtml);\n res.setStatus(HttpServletResponse.SC_OK);\n PrintWriter w = res.getWriter();\n\n int maxTime = -1;\n String maxTimeP = req.getParameter(MaxTimeParam);\n if (maxTimeP != null) {\n maxTime = Integer.parseInt(maxTimeP);\n }\n\n int introSize = 0;\n String introSizeP = req.getParameter(IntroSizeParam);\n if (introSizeP != null) {\n introSize = Integer.parseInt(introSizeP);\n }\n\n long startTime = System.currentTimeMillis();\n w.println(\"Slow Rider\");\n\n // write out a bunch of stuff at the start before interim pauses, gets past some buffers\n if (introSize != 0) {\n StringBuilder s = new StringBuilder();\n while (s.length() < introSize) {\n s.append(\"Hello and welcome to the Slow Rider!
\\n\");\n }\n w.println(s);\n w.flush();\n }\n\n while (true) {\n w.println(\"Are you still there?\");\n boolean err = w.checkError(); // flush, and check still ok\n if (err) {\n log(\"Remote connection lost\");\n break;\n }\n if (pause(SleepTime)) break;\n\n if (maxTime > 0 && System.currentTimeMillis() > startTime + maxTime) {\n w.println(\"
outatime
\");\n break;\n }\n }\n }\n\n private static boolean pause(int sleepTime) {\n try {\n Thread.sleep(sleepTime);\n } catch (InterruptedException e) {\n return true;\n }\n return false;\n }\n\n // allow the servlet to run as a main program, for local test\n public static void main(String[] args) {\n TestServer.start();\n System.out.println(Url);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/internal/ControllableInputStreamTest.java",
"content": "package org.jsoup.internal;\n\nimport org.junit.jupiter.api.Test;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.FilterInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass ControllableInputStreamTest {\n\n @Test\n void respectsMaxCapDuringFill() throws IOException {\n byte[] data = \"0123456789\".getBytes(); // 10 bytes\n CountingInputStream counting = new CountingInputStream(new ByteArrayInputStream(data));\n\n ControllableInputStream in = ControllableInputStream.wrap(counting, 5); // cap at 5 bytes\n byte[] buf = new byte[10];\n\n int read = in.read(buf);\n assertEquals(5, read, \"should only read up to cap\");\n assertEquals(5, counting.count, \"underlying stream should not be pulled past cap\");\n assertFalse(in.baseReadFully(), \"cap hit is not EOF\");\n\n int second = in.read(buf);\n assertEquals(-1, second, \"further reads return -1 once cap is exhausted\");\n assertFalse(in.baseReadFully(), \"still not true EOF\");\n in.close();\n }\n\n @Test\n void compactsBufferWithActiveMark() throws IOException {\n int size = SharedConstants.DefaultBufferSize * 2;\n byte[] data = new byte[size];\n for (int i = 0; i < size; i++) data[i] = (byte) (i % 256);\n\n ControllableInputStream in = ControllableInputStream.wrap(new ByteArrayInputStream(data), 0);\n\n byte[] first = new byte[500];\n assertEquals(500, in.read(first));\n\n in.mark(SharedConstants.DefaultBufferSize); // mark at logical pos 500\n\n byte[] consume = new byte[SharedConstants.DefaultBufferSize];\n int firstRead = in.read(consume); // serves remainder of current buffer (BufferSize - 500)\n assertEquals(SharedConstants.DefaultBufferSize - 500, firstRead);\n\n byte[] more = new byte[1000];\n int secondRead = in.read(more); // triggers fill() with active mark, then consumes from freshly filled buffer\n assertEquals(SharedConstants.DefaultBufferSize - firstRead, secondRead);\n\n in.reset(); // should rewind to mark despite prior compaction\n\n byte[] reread = new byte[1000];\n assertEquals(1000, in.read(reread));\n for (int i = 0; i < reread.length; i++) {\n assertEquals(data[500 + i], reread[i], \"byte mismatch at \" + i);\n }\n in.close();\n }\n\n private static final class CountingInputStream extends FilterInputStream {\n int count = 0;\n\n CountingInputStream(InputStream in) {\n super(in);\n }\n\n @Override\n public int read(byte[] b, int off, int len) throws IOException {\n int r = super.read(b, off, len);\n if (r > 0) count += r;\n return r;\n }\n\n @Override\n public int read() throws IOException {\n int r = super.read();\n if (r != -1) count++;\n return r;\n }\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/internal/QuietAppendableTest.java",
"content": "package org.jsoup.internal;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.SerializationException;\nimport org.jsoup.nodes.Document;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.CharArrayWriter;\nimport java.io.IOException;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass QuietAppendableTest {\n @Test void wrap() {\n assertInstanceOf(\n QuietAppendable.StringBuilderAppendable.class,\n QuietAppendable.wrap(new StringBuilder())\n );\n\n assertInstanceOf(\n QuietAppendable.BaseAppendable.class,\n QuietAppendable.wrap(new CharArrayWriter())\n );\n }\n\n @Test void supplemental() {\n // hits append(char[] chars, int offset, int len) with len 2 for supplemental codepoint\n String expect = \"😀\";\n char[] chars = new char[2];\n chars[0] = expect.charAt(0);\n chars[1] = expect.charAt(1);\n assertEquals(2, expect.length());\n\n QuietAppendable sb = QuietAppendable.wrap(new StringBuilder());\n sb.append(chars, 0, 2);\n String s = sb.toString();\n assertEquals(expect, s);\n\n CharArrayWriter cw = new CharArrayWriter();\n QuietAppendable qa = QuietAppendable.wrap(cw);\n qa.append(chars, 0, 2);\n String out = cw.toString();\n assertEquals(expect, out);\n }\n\n private static Appendable brokenAppender() {\n // returns an Appendable that throws an IOException on any put\n return new Appendable() {\n @Override\n public Appendable append(CharSequence csq) throws IOException {\n throw new IOException(\"broken\");\n }\n\n @Override\n public Appendable append(CharSequence csq, int start, int end) throws IOException {\n throw new IOException(\"broken\");\n }\n\n @Override\n public Appendable append(char c) throws IOException {\n throw new IOException(\"broken\");\n }\n };\n }\n\n @Test void appendThrowsSerializationException() {\n Document doc = Jsoup.parse(\"\");\n Appendable brokenWriter = brokenAppender();\n boolean threw = false;\n try {\n doc.html(brokenWriter);\n } catch (SerializationException e) {\n threw = true;\n Throwable cause = e.getCause();\n assertEquals(\"broken\", cause.getMessage());\n assertInstanceOf(IOException.class, cause);\n }\n assertTrue(threw);\n }\n\n}\n"
},
{
"path": "src/test/java/org/jsoup/internal/ReaderTest.java",
"content": "package org.jsoup.internal;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.parser.CharacterReader;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Test;\n\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\n\nimport static org.jsoup.integration.ParseTest.getPath;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class ReaderTest {\n @Test void readerOfStringAndFile() throws IOException {\n // make sure that reading from a String and from a File produce the same bytes\n Path path = getPath(\"/fuzztests/garble.html\");\n byte[] bytes = Files.readAllBytes(path);\n String fromBytes = new String(bytes, StandardCharsets.UTF_8);\n\n SimpleStreamReader streamReader = getReader(path);\n String fromStream = getString(streamReader);\n assertEquals(fromBytes, fromStream);\n\n SimpleStreamReader reader2 = getReader(path);\n CharacterReader cr = new CharacterReader(reader2);\n String fullRead = cr.consumeTo('X'); // does not exist in input\n assertEquals(fromBytes, fullRead);\n }\n\n private static String getString(SimpleStreamReader streamReader) throws IOException {\n // read streamreader to a string:\n StringBuilder builder = new StringBuilder();\n char[] cbuffer = new char[1024];\n int read;\n while ((read = streamReader.read(cbuffer)) != -1) {\n builder.append(cbuffer, 0, read);\n }\n return builder.toString();\n }\n\n private static SimpleStreamReader getReader(Path path) throws IOException {\n // set up a chain as in when we parse: simplebufferedinput -> controllableinputstream -> simplestreamreader -> characterreader\n SimpleBufferedInput input = new SimpleBufferedInput(Files.newInputStream(path));\n ControllableInputStream stream = ControllableInputStream.wrap(input, 0);\n return new SimpleStreamReader(stream, StandardCharsets.UTF_8);\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/internal/SoftPoolTest.java",
"content": "package org.jsoup.internal;\n\nimport org.junit.jupiter.api.Test;\n\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class SoftPoolTest {\n\n private static final int BufSize = 12;\n private static final int NumThreads = 5;\n private static final int NumObjects = 3;\n\n @Test\n public void testSoftLocalPool() throws InterruptedException {\n SoftPool
softLocalPool = new SoftPool<>(() -> new char[BufSize]);\n\n ExecutorService executorService = Executors.newFixedThreadPool(NumThreads);\n CountDownLatch latch = new CountDownLatch(NumThreads);\n\n Set allBuffers = new HashSet<>();\n Set[] threadLocalBuffers = new Set[NumThreads];\n\n for (int i = 0; i < NumThreads; i++) {\n threadLocalBuffers[i] = new HashSet<>();\n }\n\n AtomicInteger threadCount = new AtomicInteger();\n\n Runnable task = () -> {\n try {\n int threadIndex = threadCount.getAndIncrement();\n Set localBuffers = new HashSet<>();\n // First borrow\n for (int i = 0; i < NumObjects; i++) {\n char[] buffer = softLocalPool.borrow();\n assertEquals(BufSize, buffer.length);\n localBuffers.add(buffer);\n }\n\n // Release buffers back to the pool\n for (char[] buffer : localBuffers) {\n softLocalPool.release(buffer);\n }\n\n // Borrow again and ensure buffers are reused\n for (int i = 0; i < NumObjects; i++) {\n char[] buffer = softLocalPool.borrow();\n assertTrue(localBuffers.contains(buffer), \"Buffer was not reused in the same thread\");\n threadLocalBuffers[threadIndex].add(buffer);\n }\n\n synchronized (allBuffers) {\n allBuffers.addAll(threadLocalBuffers[threadIndex]);\n }\n } finally {\n latch.countDown();\n }\n };\n\n // Run the tasks\n for (int i = 0; i < NumThreads; i++) {\n executorService.submit(task::run);\n }\n\n // Wait for all threads to complete\n latch.await();\n executorService.shutdown();\n\n // Ensure no buffers are shared between threads\n Set uniqueBuffers = new HashSet<>();\n for (Set bufferSet : threadLocalBuffers) {\n for (char[] buffer : bufferSet) {\n assertTrue(uniqueBuffers.add(buffer), \"Buffer was shared between threads\");\n }\n }\n }\n\n @Test\n public void testSoftReferenceBehavior() {\n SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);\n\n // Borrow and release an object\n char[] buffer = softLocalPool.borrow();\n assertEquals(BufSize, buffer.length);\n softLocalPool.release(buffer);\n\n // Fake a GC\n softLocalPool.threadLocalStack.get().clear();\n\n // Ensure the object is garbage collected\n assertNull(softLocalPool.threadLocalStack.get().get());\n\n char[] second = softLocalPool.borrow();\n // should be different, but same size\n assertNotEquals(buffer, second);\n assertEquals(BufSize, second.length);\n }\n\n @Test\n public void testBorrowFromEmptyPool() {\n SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);\n\n // Borrow from an empty pool\n char[] buffer = softLocalPool.borrow();\n assertNotNull(buffer, \"Borrowed null from an empty pool\");\n assertEquals(BufSize, buffer.length);\n }\n\n @Test\n public void testReleaseMoreThanMaxIdle() {\n SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);\n\n // Borrow more than MaxIdle objects\n List borrowedBuffers = new ArrayList<>();\n for (int i = 0; i < SoftPool.MaxIdle + 5; i++) {\n char[] buffer = softLocalPool.borrow();\n borrowedBuffers.add(buffer);\n }\n\n // Release all borrowed objects back to the pool\n for (char[] buffer : borrowedBuffers) {\n softLocalPool.release(buffer);\n }\n\n // Ensure the pool size does not exceed MaxIdle\n ArrayDeque stack = softLocalPool.getStack();\n assertTrue(stack.size() <= SoftPool.MaxIdle, \"Pool size exceeded MaxIdle limit\");\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/internal/StringUtilTest.java",
"content": "package org.jsoup.internal;\n\nimport org.jsoup.Jsoup;\nimport org.junit.jupiter.api.Test;\n\nimport java.util.Arrays;\nimport java.util.Collections;\n\nimport static org.jsoup.internal.StringUtil.normaliseWhitespace;\nimport static org.jsoup.internal.StringUtil.resolve;\nimport static org.junit.jupiter.api.Assertions.*;\n\npublic class StringUtilTest {\n\n @Test\n public void join() {\n assertEquals(\"\", StringUtil.join(Collections.singletonList(\"\"), \" \"));\n assertEquals(\"one\", StringUtil.join(Collections.singletonList(\"one\"), \" \"));\n assertEquals(\"one two three\", StringUtil.join(Arrays.asList(\"one\", \"two\", \"three\"), \" \"));\n }\n\n @Test public void padding() {\n assertEquals(\"\", StringUtil.padding(0));\n assertEquals(\" \", StringUtil.padding(1));\n assertEquals(\" \", StringUtil.padding(2));\n assertEquals(\" \", StringUtil.padding(15));\n assertEquals(\" \", StringUtil.padding(45)); // we default to tap out at 30\n\n // memoization is up to 21 blocks (0 to 20 spaces) and exits early before min checks making maxPaddingWidth unused\n assertEquals(\"\", StringUtil.padding(0, -1));\n assertEquals(\" \", StringUtil.padding(20, -1));\n\n // this test escapes memoization and continues through\n assertEquals(\" \", StringUtil.padding(21, -1));\n\n // this test escapes memoization and using unlimited length (-1) will allow requested spaces\n assertEquals(\" \", StringUtil.padding(30, -1));\n assertEquals(\" \", StringUtil.padding(45, -1));\n\n // we tap out at 0 for this test\n assertEquals(\"\", StringUtil.padding(0, 0));\n\n // as memoization is escaped, setting zero for max padding will not allow any requested width\n assertEquals(\"\", StringUtil.padding(21, 0));\n\n // we tap out at 30 for these tests making > 30 use 30\n assertEquals(\"\", StringUtil.padding(0, 30));\n assertEquals(\" \", StringUtil.padding(1, 30));\n assertEquals(\" \", StringUtil.padding(2, 30));\n assertEquals(\" \", StringUtil.padding(15, 30));\n assertEquals(\" \", StringUtil.padding(45, 30));\n\n // max applies regardless of memoized\n assertEquals(5, StringUtil.padding(20, 5).length());\n }\n\n @Test public void paddingInACan() {\n String[] padding = StringUtil.padding;\n assertEquals(21, padding.length);\n for (int i = 0; i < padding.length; i++) {\n assertEquals(i, padding[i].length());\n }\n }\n\n @Test public void isBlank() {\n assertTrue(StringUtil.isBlank(null));\n assertTrue(StringUtil.isBlank(\"\"));\n assertTrue(StringUtil.isBlank(\" \"));\n assertTrue(StringUtil.isBlank(\" \\r\\n \"));\n\n assertFalse(StringUtil.isBlank(\"hello\"));\n assertFalse(StringUtil.isBlank(\" hello \"));\n }\n\n @Test public void isNumeric() {\n assertFalse(StringUtil.isNumeric(null));\n assertFalse(StringUtil.isNumeric(\" \"));\n assertFalse(StringUtil.isNumeric(\"123 546\"));\n assertFalse(StringUtil.isNumeric(\"hello\"));\n assertFalse(StringUtil.isNumeric(\"123.334\"));\n\n assertTrue(StringUtil.isNumeric(\"1\"));\n assertTrue(StringUtil.isNumeric(\"1234\"));\n }\n\n @Test public void isWhitespace() {\n assertTrue(StringUtil.isWhitespace('\\t'));\n assertTrue(StringUtil.isWhitespace('\\n'));\n assertTrue(StringUtil.isWhitespace('\\r'));\n assertTrue(StringUtil.isWhitespace('\\f'));\n assertTrue(StringUtil.isWhitespace(' '));\n\n assertFalse(StringUtil.isWhitespace('\\u00a0'));\n assertFalse(StringUtil.isWhitespace('\\u2000'));\n assertFalse(StringUtil.isWhitespace('\\u3000'));\n }\n\n @Test public void normaliseWhiteSpace() {\n assertEquals(\" \", normaliseWhitespace(\" \\r \\n \\r\\n\"));\n assertEquals(\" hello there \", normaliseWhitespace(\" hello \\r \\n there \\n\"));\n assertEquals(\"hello\", normaliseWhitespace(\"hello\"));\n assertEquals(\"hello there\", normaliseWhitespace(\"hello\\nthere\"));\n }\n\n @Test public void normaliseWhiteSpaceHandlesHighSurrogates() {\n String test71540chars = \"\\ud869\\udeb2\\u304b\\u309a 1\";\n String test71540charsExpectedSingleWhitespace = \"\\ud869\\udeb2\\u304b\\u309a 1\";\n\n assertEquals(test71540charsExpectedSingleWhitespace, normaliseWhitespace(test71540chars));\n String extractedText = Jsoup.parse(test71540chars).text();\n assertEquals(test71540charsExpectedSingleWhitespace, extractedText);\n }\n\n @Test public void resolvesRelativeUrls() {\n assertEquals(\"http://example.com/one/two?three\", resolve(\"http://example.com\", \"./one/two?three\"));\n assertEquals(\"http://example.com/one/two?three\", resolve(\"http://example.com?one\", \"./one/two?three\"));\n assertEquals(\"http://example.com/one/two?three#four\", resolve(\"http://example.com\", \"./one/two?three#four\"));\n assertEquals(\"https://example.com/one\", resolve(\"http://example.com/\", \"https://example.com/one\"));\n assertEquals(\"http://example.com/one/two.html\", resolve(\"http://example.com/two/\", \"../one/two.html\"));\n assertEquals(\"https://example2.com/one\", resolve(\"https://example.com/\", \"//example2.com/one\"));\n assertEquals(\"https://example.com:8080/one\", resolve(\"https://example.com:8080\", \"./one\"));\n assertEquals(\"https://example2.com/one\", resolve(\"http://example.com/\", \"https://example2.com/one\"));\n assertEquals(\"https://example.com/one\", resolve(\"wrong\", \"https://example.com/one\"));\n assertEquals(\"https://example.com/one\", resolve(\"https://example.com/one\", \"\"));\n assertEquals(\"\", resolve(\"wrong\", \"also wrong\"));\n assertEquals(\"ftp://example.com/one\", resolve(\"ftp://example.com/two/\", \"../one\"));\n assertEquals(\"ftp://example.com/one/two.c\", resolve(\"ftp://example.com/one/\", \"./two.c\"));\n assertEquals(\"ftp://example.com/one/two.c\", resolve(\"ftp://example.com/one/\", \"two.c\"));\n // examples taken from rfc3986 section 5.4.2\n assertEquals(\"http://example.com/g\", resolve(\"http://example.com/b/c/d;p?q\", \"../../../g\"));\n assertEquals(\"http://example.com/g\", resolve(\"http://example.com/b/c/d;p?q\", \"../../../../g\"));\n assertEquals(\"http://example.com/g\", resolve(\"http://example.com/b/c/d;p?q\", \"/./g\"));\n assertEquals(\"http://example.com/g\", resolve(\"http://example.com/b/c/d;p?q\", \"/../g\"));\n assertEquals(\"http://example.com/b/c/g.\", resolve(\"http://example.com/b/c/d;p?q\", \"g.\"));\n assertEquals(\"http://example.com/b/c/.g\", resolve(\"http://example.com/b/c/d;p?q\", \".g\"));\n assertEquals(\"http://example.com/b/c/g..\", resolve(\"http://example.com/b/c/d;p?q\", \"g..\"));\n assertEquals(\"http://example.com/b/c/..g\", resolve(\"http://example.com/b/c/d;p?q\", \"..g\"));\n assertEquals(\"http://example.com/b/g\", resolve(\"http://example.com/b/c/d;p?q\", \"./../g\"));\n assertEquals(\"http://example.com/b/c/g/\", resolve(\"http://example.com/b/c/d;p?q\", \"./g/.\"));\n assertEquals(\"http://example.com/b/c/g/h\", resolve(\"http://example.com/b/c/d;p?q\", \"g/./h\"));\n assertEquals(\"http://example.com/b/c/h\", resolve(\"http://example.com/b/c/d;p?q\", \"g/../h\"));\n assertEquals(\"http://example.com/b/c/g;x=1/y\", resolve(\"http://example.com/b/c/d;p?q\", \"g;x=1/./y\"));\n assertEquals(\"http://example.com/b/c/y\", resolve(\"http://example.com/b/c/d;p?q\", \"g;x=1/../y\"));\n assertEquals(\"http://example.com/b/c/g?y/./x\", resolve(\"http://example.com/b/c/d;p?q\", \"g?y/./x\"));\n assertEquals(\"http://example.com/b/c/g?y/../x\", resolve(\"http://example.com/b/c/d;p?q\", \"g?y/../x\"));\n assertEquals(\"http://example.com/b/c/g#s/./x\", resolve(\"http://example.com/b/c/d;p?q\", \"g#s/./x\"));\n assertEquals(\"http://example.com/b/c/g#s/../x\", resolve(\"http://example.com/b/c/d;p?q\", \"g#s/../x\"));\n }\n\n @Test void stripsControlCharsFromUrls() {\n // should resovle to an absolute url:\n assertEquals(\"foo:bar\", resolve(\"\\nhttps://\\texample.com/\", \"\\r\\nfo\\to:ba\\br\"));\n }\n\n @Test void allowsSpaceInUrl() {\n assertEquals(\"https://example.com/foo bar/\", resolve(\"HTTPS://example.com/example/\", \"../foo bar/\"));\n }\n\n @Test\n void isAscii() {\n assertTrue(StringUtil.isAscii(\"\"));\n assertTrue(StringUtil.isAscii(\"example.com\"));\n assertTrue(StringUtil.isAscii(\"One Two\"));\n assertFalse(StringUtil.isAscii(\"🧔\"));\n assertFalse(StringUtil.isAscii(\"测试\"));\n assertFalse(StringUtil.isAscii(\"测试.com\"));\n }\n\n @Test void isAsciiLetter() {\n assertTrue(StringUtil.isAsciiLetter('a'));\n assertTrue(StringUtil.isAsciiLetter('n'));\n assertTrue(StringUtil.isAsciiLetter('z'));\n assertTrue(StringUtil.isAsciiLetter('A'));\n assertTrue(StringUtil.isAsciiLetter('N'));\n assertTrue(StringUtil.isAsciiLetter('Z'));\n\n assertFalse(StringUtil.isAsciiLetter(' '));\n assertFalse(StringUtil.isAsciiLetter('-'));\n assertFalse(StringUtil.isAsciiLetter('0'));\n assertFalse(StringUtil.isAsciiLetter('ß'));\n assertFalse(StringUtil.isAsciiLetter('Ě'));\n }\n\n @Test void isDigit() {\n assertTrue(StringUtil.isDigit('0'));\n assertTrue(StringUtil.isDigit('1'));\n assertTrue(StringUtil.isDigit('2'));\n assertTrue(StringUtil.isDigit('3'));\n assertTrue(StringUtil.isDigit('4'));\n assertTrue(StringUtil.isDigit('5'));\n assertTrue(StringUtil.isDigit('6'));\n assertTrue(StringUtil.isDigit('7'));\n assertTrue(StringUtil.isDigit('8'));\n assertTrue(StringUtil.isDigit('9'));\n\n assertFalse(StringUtil.isDigit('a'));\n assertFalse(StringUtil.isDigit('A'));\n assertFalse(StringUtil.isDigit('ä'));\n assertFalse(StringUtil.isDigit('Ä'));\n assertFalse(StringUtil.isDigit('١'));\n assertFalse(StringUtil.isDigit('୳'));\n }\n\n @Test void isHexDigit() {\n assertTrue(StringUtil.isHexDigit('0'));\n assertTrue(StringUtil.isHexDigit('1'));\n assertTrue(StringUtil.isHexDigit('2'));\n assertTrue(StringUtil.isHexDigit('3'));\n assertTrue(StringUtil.isHexDigit('4'));\n assertTrue(StringUtil.isHexDigit('5'));\n assertTrue(StringUtil.isHexDigit('6'));\n assertTrue(StringUtil.isHexDigit('7'));\n assertTrue(StringUtil.isHexDigit('8'));\n assertTrue(StringUtil.isHexDigit('9'));\n assertTrue(StringUtil.isHexDigit('a'));\n assertTrue(StringUtil.isHexDigit('b'));\n assertTrue(StringUtil.isHexDigit('c'));\n assertTrue(StringUtil.isHexDigit('d'));\n assertTrue(StringUtil.isHexDigit('e'));\n assertTrue(StringUtil.isHexDigit('f'));\n assertTrue(StringUtil.isHexDigit('A'));\n assertTrue(StringUtil.isHexDigit('B'));\n assertTrue(StringUtil.isHexDigit('C'));\n assertTrue(StringUtil.isHexDigit('D'));\n assertTrue(StringUtil.isHexDigit('E'));\n assertTrue(StringUtil.isHexDigit('F'));\n\n assertFalse(StringUtil.isHexDigit('g'));\n assertFalse(StringUtil.isHexDigit('G'));\n assertFalse(StringUtil.isHexDigit('ä'));\n assertFalse(StringUtil.isHexDigit('Ä'));\n assertFalse(StringUtil.isHexDigit('١'));\n assertFalse(StringUtil.isHexDigit('୳'));\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/nodes/AttributeTest.java",
"content": "package org.jsoup.nodes;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.parser.ParseSettings;\nimport org.jsoup.parser.Parser;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\n\npublic class AttributeTest {\n @Test\n public void html() {\n Attribute attr = new Attribute(\"key\", \"value &\");\n assertEquals(\"key=\\\"value &\\\"\", attr.html());\n assertEquals(attr.html(), attr.toString());\n }\n\n @Test\n public void htmlWithLtAndGtInValue() {\n Attribute attr = new Attribute(\"key\", \"\");\n assertEquals(\"key=\\\"<value>\\\"\", attr.html());\n }\n\n @Test public void testWithSupplementaryCharacterInAttributeKeyAndValue() {\n String s = new String(Character.toChars(135361));\n Attribute attr = new Attribute(s, \"A\" + s + \"B\");\n assertEquals(s + \"=\\\"A\" + s + \"B\\\"\", attr.html());\n assertEquals(attr.html(), attr.toString());\n }\n\n @Test public void validatesKeysNotEmpty() {\n assertThrows(IllegalArgumentException.class, () -> new Attribute(\" \", \"Check\"));\n }\n\n @Test public void validatesKeysNotEmptyViaSet() {\n assertThrows(IllegalArgumentException.class, () -> {\n Attribute attr = new Attribute(\"One\", \"Check\");\n attr.setKey(\" \");\n });\n }\n\n @Test public void booleanAttributesAreEmptyStringValues() {\n Document doc = Jsoup.parse(\"\");\n Attributes attributes = doc.body().child(0).attributes();\n assertEquals(\"\", attributes.get(\"hidden\"));\n\n Attribute first = attributes.iterator().next();\n assertEquals(\"hidden\", first.getKey());\n assertEquals(\"\", first.getValue());\n assertFalse(first.hasDeclaredValue());\n assertTrue(Attribute.isBooleanAttribute(first.getKey()));\n }\n\n @Test public void settersOnOrphanAttribute() {\n Attribute attr = new Attribute(\"one\", \"two\");\n attr.setKey(\"three\");\n String oldVal = attr.setValue(\"four\");\n assertEquals(\"two\", oldVal);\n assertEquals(\"three\", attr.getKey());\n assertEquals(\"four\", attr.getValue());\n assertNull(attr.parent);\n }\n\n @Test void settersAfterParentRemoval() {\n // tests key and value set on a retained attribute after disconnected from parent\n Attributes attrs = new Attributes();\n attrs.put(\"foo\", \"bar\");\n Attribute attr = attrs.attribute(\"foo\");\n assertNotNull(attr);\n attrs.remove(\"foo\");\n assertEquals(\"foo\", attr.getKey());\n assertEquals(\"bar\", attr.getValue());\n attr.setKey(\"new\");\n attr.setValue(\"newer\");\n assertEquals(\"new\", attr.getKey());\n assertEquals(\"newer\", attr.getValue());\n }\n\n @Test public void hasValue() {\n Attribute a1 = new Attribute(\"one\", \"\");\n Attribute a2 = new Attribute(\"two\", null);\n Attribute a3 = new Attribute(\"thr\", \"thr\");\n\n assertTrue(a1.hasDeclaredValue());\n assertFalse(a2.hasDeclaredValue());\n assertTrue(a3.hasDeclaredValue());\n }\n\n @Test public void canSetValueToNull() {\n Attribute attr = new Attribute(\"one\", \"val\");\n String oldVal = attr.setValue(null);\n assertEquals(\"one\", attr.html());\n assertEquals(\"val\", oldVal);\n\n oldVal = attr.setValue(\"foo\");\n assertEquals(\"\", oldVal); // string, not null\n }\n\n @Test void booleanAttributesAreNotCaseSensitive() {\n // https://github.com/jhy/jsoup/issues/1656\n assertTrue(Attribute.isBooleanAttribute(\"required\"));\n assertTrue(Attribute.isBooleanAttribute(\"REQUIRED\"));\n assertTrue(Attribute.isBooleanAttribute(\"rEQUIREd\"));\n assertFalse(Attribute.isBooleanAttribute(\"random string\"));\n\n String html = \"
One\";\n Document doc = Jsoup.parse(html);\n assertEquals(\"
One\", doc.selectFirst(\"a\").outerHtml());\n\n Document doc2 = Jsoup.parse(html, Parser.htmlParser().settings(ParseSettings.preserveCase));\n assertEquals(\"
One\", doc2.selectFirst(\"a\").outerHtml());\n }\n\n @Test void orphanNamespace() {\n Attribute attr = new Attribute(\"one\", \"two\");\n assertEquals(\"\", attr.namespace());\n }\n}\n"
},
{
"path": "src/test/java/org/jsoup/nodes/AttributesTest.java",
"content": "package org.jsoup.nodes;\n\nimport org.jsoup.Jsoup;\nimport org.junit.jupiter.api.Test;\n\nimport java.util.ConcurrentModificationException;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.NoSuchElementException;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\n/**\n * Tests for Attributes.\n *\n * @author Jonathan Hedley\n */\npublic class AttributesTest {\n\n @Test\n public void html() {\n Attributes a = new Attributes();\n a.put(\"Tot\", \"a&p\");\n a.put(\"Hello\", \"There\");\n a.put(\"data-name\", \"Jsoup\");\n\n assertEquals(3, a.size());\n assertTrue(a.hasKey(\"Tot\"));\n assertTrue(a.hasKey(\"Hello\"));\n assertTrue(a.hasKey(\"data-name\"));\n assertFalse(a.hasKey(\"tot\"));\n assertTrue(a.hasKeyIgnoreCase(\"tot\"));\n assertEquals(\"There\", a.getIgnoreCase(\"hEllo\"));\n\n Map
dataset = a.dataset();\n assertEquals(1, dataset.size());\n assertEquals(\"Jsoup\", dataset.get(\"name\"));\n assertEquals(\"\", a.get(\"tot\"));\n assertEquals(\"a&p\", a.get(\"Tot\"));\n assertEquals(\"a&p\", a.getIgnoreCase(\"tot\"));\n\n assertEquals(\" Tot=\\\"a&p\\\" Hello=\\\"There\\\" data-name=\\\"Jsoup\\\"\", a.html());\n assertEquals(a.html(), a.toString());\n }\n\n @Test\n public void testIteratorRemovable() {\n Attributes a = new Attributes();\n a.put(\"Tot\", \"a&p\");\n a.put(\"Hello\", \"There\");\n a.put(\"data-name\", \"Jsoup\");\n assertTrue(a.hasKey(\"Tot\"));\n\n Iterator iterator = a.iterator();\n Attribute attr = iterator.next();\n assertEquals(\"Tot\", attr.getKey());\n iterator.remove();\n assertEquals(2, a.size());\n attr = iterator.next();\n assertEquals(\"Hello\", attr.getKey());\n assertEquals(\"There\", attr.getValue());\n\n // make sure that's flowing to the underlying attributes object\n assertEquals(2, a.size());\n assertEquals(\"There\", a.get(\"Hello\"));\n assertFalse(a.hasKey(\"Tot\"));\n }\n\n @Test\n public void testIteratorUpdateable() {\n Attributes a = new Attributes();\n a.put(\"Tot\", \"a&p\");\n a.put(\"Hello\", \"There\");\n\n assertFalse(a.hasKey(\"Foo\"));\n Iterator iterator = a.iterator();\n Attribute attr = iterator.next();\n attr.setKey(\"Foo\");\n attr = iterator.next();\n attr.setKey(\"Bar\");\n attr.setValue(\"Qux\");\n\n assertEquals(\"a&p\", a.get(\"Foo\"));\n assertEquals(\"Qux\", a.get(\"Bar\"));\n assertFalse(a.hasKey(\"Tot\"));\n assertFalse(a.hasKey(\"Hello\"));\n }\n\n @Test public void testIteratorHasNext() {\n Attributes a = new Attributes();\n a.put(\"Tot\", \"1\");\n a.put(\"Hello\", \"2\");\n a.put(\"data-name\", \"3\");\n\n int seen = 0;\n for (Attribute attribute : a) {\n seen++;\n assertEquals(String.valueOf(seen), attribute.getValue());\n }\n assertEquals(3, seen);\n }\n\n @Test\n public void testIterator() {\n Attributes a = new Attributes();\n String[][] datas = {{\"Tot\", \"raul\"},\n {\"Hello\", \"pismuth\"},\n {\"data-name\", \"Jsoup\"}};\n for (String[] atts : datas) {\n a.put(atts[0], atts[1]);\n }\n\n Iterator iterator = a.iterator();\n assertTrue(iterator.hasNext());\n int i = 0;\n for (Attribute attribute : a) {\n assertEquals(datas[i][0], attribute.getKey());\n assertEquals(datas[i][1], attribute.getValue());\n i++;\n }\n assertEquals(datas.length, i);\n }\n\n @Test\n public void testIteratorSkipsInternal() {\n Attributes a = new Attributes();\n a.put(\"One\", \"One\");\n a.put(Attributes.internalKey(\"baseUri\"), \"example.com\");\n a.put(\"Two\", \"Two\");\n a.put(Attributes.internalKey(\"another\"), \"example.com\");\n\n Iterator it = a.iterator();\n assertTrue(it.hasNext());\n assertEquals(\"One\", it.next().getKey());\n assertTrue(it.hasNext());\n assertEquals(\"Two\", it.next().getKey());\n assertFalse(it.hasNext());\n\n int seen = 0;\n for (Attribute attribute : a) {\n seen++;\n }\n assertEquals(2, seen);\n }\n\n @Test void iteratorThrows() {\n Attributes attrs = new Attributes();\n attrs.put(\"One\", \"one\").put(\"Two\", \"two\");\n\n Iterator it = attrs.iterator();\n int seen = 0;\n while (it.hasNext()) {\n it.next();\n seen++;\n }\n assertFalse(it.hasNext());\n assertEquals(2, seen);\n\n boolean threw = false;\n try {\n Attribute next = it.next();\n assertNotNull(next); // not hit\n } catch (NoSuchElementException e) {\n threw = true;\n }\n assertTrue(threw);\n }\n\n @Test\n public void testListSkipsInternal() {\n Attributes a = new Attributes();\n a.put(\"One\", \"One\");\n a.put(Attributes.internalKey(\"baseUri\"), \"example.com\");\n a.put(\"Two\", \"Two\");\n a.put(Attributes.internalKey(\"another\"), \"example.com\");\n\n List attributes = a.asList();\n assertEquals(2, attributes.size());\n assertEquals(\"One\", attributes.get(0).getKey());\n assertEquals(\"Two\", attributes.get(1). getKey());\n }\n\n @Test public void htmlSkipsInternals() {\n Attributes a = new Attributes();\n a.put(\"One\", \"One\");\n a.put(Attributes.internalKey(\"baseUri\"), \"example.com\");\n a.put(\"Two\", \"Two\");\n a.put(Attributes.internalKey(\"another\"), \"example.com\");\n\n assertEquals(\" One=\\\"One\\\" Two=\\\"Two\\\"\", a.html());\n }\n\n @Test\n public void testIteratorEmpty() {\n Attributes a = new Attributes();\n\n Iterator iterator = a.iterator();\n assertFalse(iterator.hasNext());\n }\n\n @Test\n public void testIteratorRemove() {\n String html = \"