Each Node is still the same DOM connected Node.
@return a new ArrayList containing the nodes in this list
@see #Nodes(List)
*/
public ArrayList asList() {
return new ArrayList<>(this);
}
/**
Remove each matched node from the DOM.
The nodes will still be retained in this list, in case further processing of them is desired.
E.g. HTML: {@code
Hello
there
![]()
doc.select("p").remove();
HTML = {@code ![]()
}
Note that this method should not be used to clean user-submitted HTML; rather, use {@link org.jsoup.safety.Cleaner}
to clean HTML.
@return this, for chaining
@see Element#empty()
@see Elements#empty()
@see #clear()
*/
public Nodes remove() {
for (T node : this) {
node.remove();
}
return this;
}
/**
Get the combined outer HTML of all matched nodes.
@return string of all node's outer HTML.
@see Elements#text()
@see Elements#html()
*/
public String outerHtml() {
return stream()
.map(Node::outerHtml)
.collect(StringUtil.joining("\n"));
}
/**
Get the combined outer HTML of all matched nodes. Alias of {@link #outerHtml()}.
@return string of all the node's outer HTML.
@see Elements#text()
@see #outerHtml()
*/
@Override
public String toString() {
return outerHtml();
}
/**
Insert the supplied HTML before each matched node's outer HTML.
@param html HTML to insert before each node
@return this, for chaining
@see Element#before(String)
*/
public Nodes before(String html) {
for (T node : this) {
node.before(html);
}
return this;
}
/**
Insert the supplied HTML after each matched nodes's outer HTML.
@param html HTML to insert after each node
@return this, for chaining
@see Element#after(String)
*/
public Nodes after(String html) {
for (T node : this) {
node.after(html);
}
return this;
}
/**
Wrap the supplied HTML around each matched node. For example, with HTML
{@code This is Jsoup
},
doc.select("b").wrap("<i></i>");
becomes {@code This is jsoup
}
@param html HTML to wrap around each node, e.g. {@code }. Can be arbitrarily deep.
@return this (for chaining)
@see Element#wrap
*/
public Nodes wrap(String html) {
Validate.notEmpty(html);
for (T node : this) {
node.wrap(html);
}
return this;
}
// list-like methods
/**
Get the first matched element.
@return The first matched element, or null if contents is empty.
*/
public @Nullable T first() {
return isEmpty() ? null : get(0);
}
/**
Get the last matched element.
@return The last matched element, or null if contents is empty.
*/
public @Nullable T last() {
return isEmpty() ? null : get(size() - 1);
}
// ArrayList methods that update the DOM:
/**
Replace the node at the specified index in this list, and in the DOM.
@param index index of the node to replace
@param node node to be stored at the specified position
@return the old Node at this index
*/
@Override
public T set(int index, T node) {
Validate.notNull(node);
T old = super.set(index, node);
old.replaceWith(node);
return old;
}
/**
Remove the node at the specified index in this list, and from the DOM.
@param index the index of the node to be removed
@return the old node at this index
@see #deselect(int)
*/
@Override
public T remove(int index) {
T old = super.remove(index);
old.remove();
return old;
}
/**
Remove the specified node from this list, and from the DOM.
@param o node to be removed from this list, if present
@return if this list contained the Node
@see #deselect(Object)
*/
@Override
public boolean remove(Object o) {
int index = super.indexOf(o);
if (index == -1) {
return false;
} else {
remove(index);
return true;
}
}
/**
Remove the node at the specified index in this list, but not from the DOM.
@param index the index of the node to be removed
@return the old node at this index
@see #remove(int)
*/
public T deselect(int index) {
return super.remove(index);
}
/**
Remove the specified node from this list, but not from the DOM.
@param o node to be removed from this list, if present
@return if this list contained the Node
@see #remove(Object)
*/
public boolean deselect(Object o) {
return super.remove(o);
}
/**
Removes all the nodes from this list, and each of them from the DOM.
@see #deselectAll()
*/
@Override
public void clear() {
remove();
super.clear();
}
/**
Like {@link #clear()}, removes all the nodes from this list, but not from the DOM.
@see #clear()
*/
public void deselectAll() {
super.clear();
}
/**
Removes from this list, and from the DOM, each of the nodes that are contained in the specified collection and are
in this list.
@param c collection containing nodes to be removed from this list
@return {@code true} if nodes were removed from this list
*/
@Override
public boolean removeAll(Collection c) {
boolean anyRemoved = false;
for (Object o : c) {
anyRemoved |= this.remove(o);
}
return anyRemoved;
}
/**
Retain in this list, and in the DOM, only the nodes that are in the specified collection and are in this list. In
other words, remove nodes from this list and the DOM any item that is in this list but not in the specified
collection.
@param toRemove collection containing nodes to be retained in this list
@return {@code true} if nodes were removed from this list
@since 1.17.1
*/
@Override
public boolean retainAll(Collection toRemove) {
boolean anyRemoved = false;
for (Iterator it = this.iterator(); it.hasNext(); ) {
T el = it.next();
if (!toRemove.contains(el)) {
it.remove();
anyRemoved = true;
}
}
return anyRemoved;
}
/**
Remove from the list, and from the DOM, all nodes in this list that mach the given predicate.
@param filter a predicate which returns {@code true} for nodes to be removed
@return {@code true} if nodes were removed from this list
*/
@Override
public boolean removeIf(Predicate filter) {
boolean anyRemoved = false;
for (Iterator it = this.iterator(); it.hasNext(); ) {
T node = it.next();
if (filter.test(node)) {
it.remove();
anyRemoved = true;
}
}
return anyRemoved;
}
/**
Replace each node in this list with the result of the operator, and update the DOM.
@param operator the operator to apply to each node
*/
@Override
public void replaceAll(UnaryOperator operator) {
for (int i = 0; i < this.size(); i++) {
this.set(i, operator.apply(this.get(i)));
}
}
}
================================================
FILE: src/main/java/org/jsoup/select/QueryParser.java
================================================
package org.jsoup.select;
import org.jsoup.helper.Regex;
import org.jsoup.internal.StringUtil;
import org.jsoup.helper.Validate;
import org.jsoup.nodes.CDataNode;
import org.jsoup.nodes.Comment;
import org.jsoup.nodes.DataNode;
import org.jsoup.nodes.LeafNode;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.parser.TokenQueue;
import org.jspecify.annotations.Nullable;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.jsoup.select.StructuralEvaluator.ImmediateParentRun;
import static org.jsoup.internal.Normalizer.normalize;
/**
* Parses a CSS selector into an Evaluator tree.
*/
public class QueryParser implements AutoCloseable {
private final static char[] Combinators = {'>', '+', '~'}; // ' ' is also a combinator, but found implicitly
private final static String[] AttributeEvals = new String[]{"=", "!=", "^=", "$=", "*=", "~="};
private final static char[] SequenceEnders = {',', ')'};
private final TokenQueue tq;
private final String query;
private boolean inNodeContext; // ::comment:contains should act on node value, vs element text
/**
* Create a new QueryParser.
* @param query CSS query
*/
private QueryParser(String query) {
Validate.notEmpty(query);
query = query.trim();
this.query = query;
this.tq = new TokenQueue(query);
}
/**
Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to
parse it once and reuse the Evaluator.
@param query CSS query
@return Evaluator
@see Selector selector query syntax
@throws Selector.SelectorParseException if the CSS query is invalid
*/
public static Evaluator parse(String query) {
try (QueryParser p = new QueryParser(query)) {
return p.parse();
} catch (IllegalArgumentException e) {
throw new Selector.SelectorParseException(e.getMessage());
}
}
/**
Parse the query. We use this simplified expression of the grammar:
SelectorGroup ::= Selector (',' Selector)*
Selector ::= [ Combinator ] SimpleSequence ( Combinator SimpleSequence )*
SimpleSequence ::= [ TypeSelector ] ( ID | Class | Attribute | Pseudo )*
Pseudo ::= ':' Name [ '(' SelectorGroup ')' ]
Combinator ::= S+ // descendant (whitespace)
| '>' // child
| '+' // adjacent sibling
| '~' // general sibling
See selectors-4 for the real thing
*/
Evaluator parse() {
Evaluator eval = parseSelectorGroup();
tq.consumeWhitespace();
if (!tq.isEmpty())
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
return eval;
}
Evaluator parseSelectorGroup() {
// SelectorGroup. Into an Or if > 1 Selector
Evaluator left = parseSelector();
while (tq.matchChomp(',')) {
Evaluator right = parseSelector();
left = or(left, right);
}
return left;
}
Evaluator parseSelector() {
// Selector ::= [ Combinator ] SimpleSequence ( Combinator SimpleSequence )*
tq.consumeWhitespace();
Evaluator left;
if (tq.matchesAny(Combinators)) {
// e.g. query is "> div"; left side is root element
left = new StructuralEvaluator.Root();
} else {
left = parseSimpleSequence();
}
while (true) {
char combinator = 0;
if (tq.consumeWhitespace())
combinator = ' '; // maybe descendant?
if (tq.matchesAny(Combinators)) // no, explicit
combinator = tq.consume();
else if (tq.matchesAny(SequenceEnders)) // , - space after simple like "foo , bar"; ) - close of :has()
break;
if (combinator != 0) {
Evaluator right = parseSimpleSequence();
left = combinator(left, combinator, right);
} else {
break;
}
}
return left;
}
Evaluator parseSimpleSequence() {
// SimpleSequence ::= TypeSelector? ( Hash | Class | Pseudo )*
Evaluator left = null;
tq.consumeWhitespace();
// one optional type selector
if (tq.matchesWord() || tq.matches("*|"))
left = byTag();
else if (tq.matchChomp('*'))
left = new Evaluator.AllElements();
// zero or more subclasses (#, ., [)
while(true) {
Evaluator right = parseSubclass();
if (right != null) {
left = and(left, right);
}
else break; // no more simple tokens
}
if (left == null)
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
return left;
}
static Evaluator combinator(Evaluator left, char combinator, Evaluator right) {
switch (combinator) {
case '>':
ImmediateParentRun run = left instanceof ImmediateParentRun ?
(ImmediateParentRun) left : new ImmediateParentRun(left);
run.add(right);
return run;
case ' ':
return and(new StructuralEvaluator.Ancestor(left), right);
case '+':
return and(new StructuralEvaluator.ImmediatePreviousSibling(left), right);
case '~':
return and(new StructuralEvaluator.PreviousSibling(left), right);
default:
throw new Selector.SelectorParseException("Unknown combinator '%s'", combinator);
}
}
@Nullable Evaluator parseSubclass() {
// Subclass: ID | Class | Attribute | Pseudo
if (tq.matchChomp('#')) return byId();
else if (tq.matchChomp('.')) return byClass();
else if (tq.matches('[')) return byAttribute();
else if (tq.matchChomp("::")) return parseNodeSelector(); // ::comment etc
else if (tq.matchChomp(':')) return parsePseudoSelector();
else return null;
}
/** Merge two evals into an Or. */
static Evaluator or(Evaluator left, Evaluator right) {
if (left instanceof CombiningEvaluator.Or) {
((CombiningEvaluator.Or) left).add(right);
return left;
}
return new CombiningEvaluator.Or(left, right);
}
/** Merge two evals into an And. */
static Evaluator and(@Nullable Evaluator left, Evaluator right) {
if (left == null) return right;
if (left instanceof CombiningEvaluator.And) {
((CombiningEvaluator.And) left).add(right);
return left;
}
return new CombiningEvaluator.And(left, right);
}
private Evaluator parsePseudoSelector() {
final String pseudo = tq.consumeCssIdentifier();
switch (pseudo) {
case "lt":
return new Evaluator.IndexLessThan(consumeIndex());
case "gt":
return new Evaluator.IndexGreaterThan(consumeIndex());
case "eq":
return new Evaluator.IndexEquals(consumeIndex());
case "has":
return has();
case "is":
return is();
case "contains":
return contains(false);
case "containsOwn":
return contains(true);
case "containsWholeText":
return containsWholeText(false);
case "containsWholeOwnText":
return containsWholeText(true);
case "containsData":
return containsData();
case "matches":
return matches(false);
case "matchesOwn":
return matches(true);
case "matchesWholeText":
return matchesWholeText(false);
case "matchesWholeOwnText":
return matchesWholeText(true);
case "not":
return not();
case "nth-child":
return cssNthChild(false, false);
case "nth-last-child":
return cssNthChild(true, false);
case "nth-of-type":
return cssNthChild(false, true);
case "nth-last-of-type":
return cssNthChild(true, true);
case "first-child":
return new Evaluator.IsFirstChild();
case "last-child":
return new Evaluator.IsLastChild();
case "first-of-type":
return new Evaluator.IsFirstOfType();
case "last-of-type":
return new Evaluator.IsLastOfType();
case "only-child":
return new Evaluator.IsOnlyChild();
case "only-of-type":
return new Evaluator.IsOnlyOfType();
case "empty":
return new Evaluator.IsEmpty();
case "blank":
return new NodeEvaluator.BlankValue();
case "root":
return new Evaluator.IsRoot();
case "matchText":
return new Evaluator.MatchText();
default:
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
}
}
// ::comment etc
private Evaluator parseNodeSelector() {
final String pseudo = tq.consumeCssIdentifier();
inNodeContext = true; // Enter node context
Evaluator left;
switch (pseudo) {
case "node":
left = new NodeEvaluator.InstanceType(Node.class, pseudo);
break;
case "leafnode":
left = new NodeEvaluator.InstanceType(LeafNode.class, pseudo);
break;
case "text":
left = new NodeEvaluator.InstanceType(TextNode.class, pseudo);
break;
case "comment":
left = new NodeEvaluator.InstanceType(Comment.class, pseudo);
break;
case "data":
left = new NodeEvaluator.InstanceType(DataNode.class, pseudo);
break;
case "cdata":
left = new NodeEvaluator.InstanceType(CDataNode.class, pseudo);
break;
default:
throw new Selector.SelectorParseException(
"Could not parse query '%s': unknown node type '::%s'", query, pseudo);
}
// Handle following subclasses in node context (like ::comment:contains())
Evaluator right;
while ((right = parseSubclass()) != null) {
left = and(left, right);
}
inNodeContext = false;
return left;
}
private Evaluator byId() {
String id = tq.consumeCssIdentifier();
Validate.notEmpty(id);
return new Evaluator.Id(id);
}
private Evaluator byClass() {
String className = tq.consumeCssIdentifier();
Validate.notEmpty(className);
return new Evaluator.Class(className.trim());
}
private Evaluator byTag() {
// todo - these aren't dealing perfectly with case sensitivity. For case sensitive parsers, we should also make
// the tag in the selector case-sensitive (and also attribute names). But for now, normalize (lower-case) for
// consistency - both the selector and the element tag
String tagName = normalize(tq.consumeElementSelector());
Validate.notEmpty(tagName);
// namespaces:
if (tagName.startsWith("*|")) { // namespaces: wildcard match equals(tagName) or ending in ":"+tagName
String plainTag = tagName.substring(2); // strip *|
return new CombiningEvaluator.Or(
new Evaluator.Tag(plainTag),
new Evaluator.TagEndsWith(":" + plainTag)
);
} else if (tagName.endsWith("|*")) { // ns|*
String ns = tagName.substring(0, tagName.length() - 2) + ":"; // strip |*, to ns:
return new Evaluator.TagStartsWith(ns);
} else if (tagName.contains("|")) { // flip "abc|def" to "abc:def"
tagName = tagName.replace("|", ":");
}
return new Evaluator.Tag(tagName);
}
private Evaluator byAttribute() {
try (TokenQueue cq = new TokenQueue(tq.chompBalanced('[', ']'))) {
return evaluatorForAttribute(cq);
}
}
private Evaluator evaluatorForAttribute(TokenQueue cq) {
String key = cq.consumeToAny(AttributeEvals); // eq, not, start, end, contain, match, (no val)
key = normalize(key);
Validate.notEmpty(key);
Validate.isFalse(key.equals("abs:"), "Absolute attribute key must have a name");
cq.consumeWhitespace();
final Evaluator eval;
if (cq.isEmpty()) {
if (key.startsWith("^"))
eval = new Evaluator.AttributeStarting(key.substring(1));
else if (key.equals("*")) // any attribute
eval = new Evaluator.AttributeStarting("");
else
eval = new Evaluator.Attribute(key);
} else {
if (cq.matchChomp('='))
eval = new Evaluator.AttributeWithValue(key, cq.remainder());
else if (cq.matchChomp("!="))
eval = new Evaluator.AttributeWithValueNot(key, cq.remainder());
else if (cq.matchChomp("^="))
eval = new Evaluator.AttributeWithValueStarting(key, cq.remainder());
else if (cq.matchChomp("$="))
eval = new Evaluator.AttributeWithValueEnding(key, cq.remainder());
else if (cq.matchChomp("*="))
eval = new Evaluator.AttributeWithValueContaining(key, cq.remainder());
else if (cq.matchChomp("~="))
eval = new Evaluator.AttributeWithValueMatching(key, Regex.compile(cq.remainder()));
else
throw new Selector.SelectorParseException(
"Could not parse attribute query '%s': unexpected token at '%s'", query, cq.remainder());
}
return eval;
}
//pseudo selectors :first-child, :last-child, :nth-child, ...
private static final Pattern NthStepOffset = Pattern.compile("(([+-])?(\\d+)?)n(\\s*([+-])?\\s*\\d+)?", Pattern.CASE_INSENSITIVE);
private static final Pattern NthOffset = Pattern.compile("([+-])?(\\d+)");
private Evaluator cssNthChild(boolean last, boolean ofType) {
String arg = normalize(consumeParens()); // arg is like "odd", or "-n+2", within nth-child(odd)
final int step, offset;
if ("odd".equals(arg)) {
step = 2;
offset = 1;
} else if ("even".equals(arg)) {
step = 2;
offset = 0;
} else {
Matcher stepOffsetM, stepM;
if ((stepOffsetM = NthStepOffset.matcher(arg)).matches()) {
if (stepOffsetM.group(3) != null) // has digits, like 3n+2 or -3n+2
step = Integer.parseInt(stepOffsetM.group(1).replaceFirst("^\\+", ""));
else // no digits, might be like n+2, or -n+2. if group(2) == "-", it’s -1;
step = "-".equals(stepOffsetM.group(2)) ? -1 : 1;
offset =
stepOffsetM.group(4) != null ? Integer.parseInt(stepOffsetM.group(4).replaceFirst("^\\+", "")) : 0;
} else if ((stepM = NthOffset.matcher(arg)).matches()) {
step = 0;
offset = Integer.parseInt(stepM.group().replaceFirst("^\\+", ""));
} else {
throw new Selector.SelectorParseException("Could not parse nth-index '%s': unexpected format", arg);
}
}
return ofType
? (last ? new Evaluator.IsNthLastOfType(step, offset) : new Evaluator.IsNthOfType(step, offset))
: (last ? new Evaluator.IsNthLastChild(step, offset) : new Evaluator.IsNthChild(step, offset));
}
private String consumeParens() {
return tq.chompBalanced('(', ')');
}
private int consumeIndex() {
String index = consumeParens().trim();
Validate.isTrue(StringUtil.isNumeric(index), "Index must be numeric");
return Integer.parseInt(index);
}
// pseudo selector :has(el)
private Evaluator has() {
return parseNested(StructuralEvaluator.Has::new, ":has() must have a selector");
}
// pseudo selector :is()
private Evaluator is() {
return parseNested(StructuralEvaluator.Is::new, ":is() must have a selector");
}
private Evaluator parseNested(Function func, String err) {
Validate.isTrue(tq.matchChomp('('), err);
Evaluator eval = parseSelectorGroup();
Validate.isTrue(tq.matchChomp(')'), err);
return func.apply(eval);
}
// pseudo selector :contains(text), containsOwn(text)
private Evaluator contains(boolean own) {
String query = own ? ":containsOwn" : ":contains";
String searchText = TokenQueue.unescape(consumeParens());
Validate.notEmpty(searchText, query + "(text) query must not be empty");
if (inNodeContext)
return new NodeEvaluator.ContainsValue(searchText);
return own
? new Evaluator.ContainsOwnText(searchText)
: new Evaluator.ContainsText(searchText);
}
private Evaluator containsWholeText(boolean own) {
String query = own ? ":containsWholeOwnText" : ":containsWholeText";
String searchText = TokenQueue.unescape(consumeParens());
Validate.notEmpty(searchText, query + "(text) query must not be empty");
return own
? new Evaluator.ContainsWholeOwnText(searchText)
: new Evaluator.ContainsWholeText(searchText);
}
// pseudo selector :containsData(data)
private Evaluator containsData() {
String searchText = TokenQueue.unescape(consumeParens());
Validate.notEmpty(searchText, ":containsData(text) query must not be empty");
return new Evaluator.ContainsData(searchText);
}
// :matches(regex), matchesOwn(regex)
private Evaluator matches(boolean own) {
String query = own ? ":matchesOwn" : ":matches";
String regex = consumeParens(); // don't unescape, as regex bits will be escaped
Validate.notEmpty(regex, query + "(regex) query must not be empty");
Regex pattern = Regex.compile(regex);
if (inNodeContext)
return new NodeEvaluator.MatchesValue(pattern);
return own
? new Evaluator.MatchesOwn(pattern)
: new Evaluator.Matches(pattern);
}
// :matches(regex), matchesOwn(regex)
private Evaluator matchesWholeText(boolean own) {
String query = own ? ":matchesWholeOwnText" : ":matchesWholeText";
String regex = consumeParens(); // don't unescape, as regex bits will be escaped
Validate.notEmpty(regex, query + "(regex) query must not be empty");
Regex pattern = Regex.compile(regex);
return own
? new Evaluator.MatchesWholeOwnText(pattern)
: new Evaluator.MatchesWholeText(pattern);
}
// :not(selector)
private Evaluator not() {
String subQuery = consumeParens();
Validate.notEmpty(subQuery, ":not(selector) subselect must not be empty");
return new StructuralEvaluator.Not(parse(subQuery));
}
@Override
public String toString() {
return query;
}
@Override
public void close() {
tq.close();
}
}
================================================
FILE: src/main/java/org/jsoup/select/Selector.java
================================================
package org.jsoup.select;
import org.jsoup.helper.Validate;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.parser.TokenQueue;
import org.jspecify.annotations.Nullable;
import java.util.Collection;
import java.util.HashSet;
import java.util.stream.Stream;
/**
CSS element selector, that finds elements matching a query.
Selector syntax
A selector is a chain of simple selectors, separated by combinators. Selectors are case-insensitive (including
against elements, attributes, and attribute values).
The universal selector {@code *} is implicit when no element selector is supplied (i.e. {@code .header} and
{@code *.header} are equivalent).
You can easily test different selectors using the Try jsoup online playground.
| Pattern | Matches | Example |
|---|
* | any element | * |
tag | elements with the given tag name | div |
*|E | elements of type E in any namespace (including non-namespaced) | *|name finds <dc:name> and <name> elements |
ns|E | elements of type E in the namespace ns | dc|name finds <dc:name> elements |
ns|* | all elements in the namespace ns | dc|* finds <dc:p> and <dc:img>elements |
#id | elements with attribute ID of "id" | div#wrap, #logo |
.class | elements with a class name of "class" | div.left, .result |
[attr] | elements with an attribute named "attr" (with any value) | a[href], [title] |
[^attrPrefix] | elements with an attribute name starting with "attrPrefix". Use to find elements with HTML5 datasets | [^data-], div[^data-] |
[attr=val] | elements with an attribute named "attr", and value equal to "val" | img[width=500], a[rel=nofollow] |
[attr="val"] | elements with an attribute named "attr", and value equal to "val" | span[hello="Cleveland"][goodbye="Columbus"], a[rel="nofollow"] |
[attr^=valPrefix] | elements with an attribute named "attr", and value starting with "valPrefix" | a[href^=http:] |
[attr$=valSuffix] | elements with an attribute named "attr", and value ending with "valSuffix" | img[src$=.png] |
[attr*=valContaining] | elements with an attribute named "attr", and value containing "valContaining" | a[href*=/search/] |
[attr~=regex] | elements with an attribute named "attr", and value matching the regular expression | img[src~=(?i)\\.(png|jpe?g)] |
[*] | elements with any attribute | p[*] finds p elements that have at least one attribute; p:not([*]) finds those with no attributes |
| The above may be combined in any order | div.header[title] |
Combinators |
E F | an F element descended from an E element | div a, .logo h1 |
E {@literal >} F | an F direct child of E | ol {@literal >} li |
E + F | an F element immediately preceded by sibling E | li + li, div.head + div |
E ~ F | an F element preceded by sibling E | h1 ~ p |
E, F, G | all matching elements E, F, or G | a[href], div, h3 |
Pseudo selectors |
:lt(n) | elements whose sibling index is less than n | td:lt(3) finds the first 3 cells of each row |
:gt(n) | elements whose sibling index is greater than n | td:gt(1) finds cells after skipping the first two |
:eq(n) | elements whose sibling index is equal to n | td:eq(0) finds the first cell of each row |
:has(selector) | elements that contains at least one element matching the selector | div:has(p) finds divs that contain p elements.
div:has(> a) selects div elements that have at least one direct child a element.
section:has(h1, h2) finds section elements that contain a h1 or a h2 element |
:is(selector list) | elements that match any of the selectors in the selector list | :is(h1, h2, h3, h4, h5, h6) finds any heading element.
:is(section, article) > :is(h1, h2) finds a h1 or h2 that is a direct child of a section or an article |
:not(selector) | elements that do not match the selector. See also {@link Elements#not(String)} | div:not(.logo) finds all divs that do not have the "logo" class.div:not(:has(div)) finds divs that do not contain divs.
|
:contains(text) | elements that contains the specified text. The search is case insensitive. The text may appear in the found element, or any of its descendants. The text is whitespace normalized. To find content that includes parentheses, escape those with a {@code \}. | p:contains(jsoup) finds p elements containing the text "jsoup".{@code p:contains(hello \(there\) finds p elements containing the text "Hello (There)"} |
:containsOwn(text) | elements that directly contain the specified text. The search is case insensitive. The text must appear in the found element, not any of its descendants. | p:containsOwn(jsoup) finds p elements with own text "jsoup". |
:containsData(data) | elements that contains the specified data. The contents of {@code script} and {@code style} elements, and {@code comment} nodes (etc) are considered data nodes, not text nodes. The search is case insensitive. The data may appear in the found element, or any of its descendants. | script:contains(jsoup) finds script elements containing the data "jsoup". |
:containsWholeText(text) | elements that contains the specified non-normalized text. The search is case sensitive, and will match exactly against spaces and newlines found in the original input. The text may appear in the found element, or any of its descendants. To find content that includes parentheses, escape those with a {@code \}. | p:containsWholeText(jsoup\nThe Java HTML Parser) finds p elements containing the text "jsoup\nThe Java HTML Parser" (and not other variations of whitespace or casing, as :contains() would. Note that {@code br} elements are presented as a newline. |
:containsWholeOwnText(text) | elements that directly contain the specified non-normalized text. The search is case sensitive, and will match exactly against spaces and newlines found in the original input. The text may appear in the found element, but not in its descendants. To find content that includes parentheses, escape those with a {@code \}. | p:containsWholeOwnText(jsoup\nThe Java HTML Parser) finds p elements directly containing the text "jsoup\nThe Java HTML Parser" (and not other variations of whitespace or casing, as :contains() would. Note that {@code br} elements are presented as a newline. |
:matches(regex) | elements containing whitespace normalized text that matches the specified regular expression. The text may appear in the found element, or any of its descendants. | td:matches(\\d+) finds table cells containing digits. div:matches((?i)login) finds divs containing the text, case insensitively. |
:matchesWholeText(regex) | elements containing non-normalized whole text that matches the specified regular expression. The text may appear in the found element, or any of its descendants. | td:matchesWholeText(\\s{2,}) finds table cells a run of at least two space characters. |
:matchesWholeOwnText(regex) | elements whose own non-normalized whole text matches the specified regular expression. The text must appear in the found element, not any of its descendants. | td:matchesWholeOwnText(\n\\d+) finds table cells directly containing digits following a neewline. |
| The above may be combined in any order and with other selectors | .light:contains(name):eq(0) |
:matchText | treats text nodes as elements, and so allows you to match against and select text nodes. Note that using this selector will modify the DOM, so you may want to {@code clone} your document before using. Deprecated. This selector is deprecated and will be removed in a future version. Migrate to ::textnode using the Element#selectNodes() method instead. | {@code p:matchText:firstChild} with input {@code One
Two } will return one {@link org.jsoup.nodes.PseudoTextElement} with text "{@code One}". |
Structural pseudo selectors |
:root | The element that is the root of the document. In HTML, this is the html element | :root |
:nth-child(an+b) | elements that have an+b-1 siblings before it in the document tree, for any positive integer or zero value of n, and has a parent element. For values of a and b greater than zero, this effectively divides the element's children into groups of a elements (the last group taking the remainder), and selecting the bth element of each group. For example, this allows the selectors to address every other row in a table, and could be used to alternate the color of paragraph text in a cycle of four. The a and b values must be integers (positive, negative, or zero). The index of the first child of an element is 1.
Additionally, :nth-child() supports odd and even as arguments. odd is the same as 2n+1, and even is the same as 2n. | tr:nth-child(2n+1) finds every odd row of a table. :nth-child(10n-1) the 9th, 19th, 29th, etc, element. li:nth-child(5) the 5h li |
:nth-last-child(an+b) | elements that have an+b-1 siblings after it in the document tree. Otherwise like :nth-child() | tr:nth-last-child(-n+2) the last two rows of a table |
:nth-of-type(an+b) | pseudo-class notation represents an element that has an+b-1 siblings with the same expanded element name before it in the document tree, for any zero or positive integer value of n, and has a parent element | img:nth-of-type(2n+1) |
:nth-last-of-type(an+b) | pseudo-class notation represents an element that has an+b-1 siblings with the same expanded element name after it in the document tree, for any zero or positive integer value of n, and has a parent element | img:nth-last-of-type(2n+1) |
:first-child | elements that are the first child of some other element. | div {@literal >} p:first-child |
:last-child | elements that are the last child of some other element. | ol {@literal >} li:last-child |
:first-of-type | elements that are the first sibling of its type in the list of children of its parent element | dl dt:first-of-type |
:last-of-type | elements that are the last sibling of its type in the list of children of its parent element | tr {@literal >} td:last-of-type |
:only-child | elements that have a parent element and whose parent element have no other element children | |
:only-of-type | an element that has a parent element and whose parent element has no other element children with the same expanded element name | |
:empty | elements that contain no child elements or nodes, with the exception of blank text nodes, comments, XML declarations, and doctype declarations. In other words, it matches elements that are effectively empty of meaningful content. | li:not(:empty) |
Node pseudo selectors |
These selectors enable matching specific leaf nodes, including Comments, TextNodes. When used with {@link Element#select(String)}, these can be used with structural selectors such as :has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@Element#selectNodes(String)}. |
| ::node | Matches any node | |
| ::leafnode | Matches any leaf-node (this is, a Node which is not an Element) | |
| ::comment | Matches a Comment node | |
| ::text | Matches a TextNode | |
| ::data | Matches a DataNode (e.g. the content of a script or a style element) | |
| ::cdata | Matches a CDataNode (which are only present in XML) | |
| ::node:contains(text) | Matches a node that has a (normalized, case-insensitive) value containing text. | ::comment:contains(foo bar) |
| ::node:matches(regex) | Matches a node that has a value matching the regex. | ::comment:matches(\\d+) |
| ::node:blank | Matches a node that has either no value, or a value of only whitespace. | ::comment:not(:blank) |
A word on using regular expressions in these selectors: depending on the content of the regex, you will need to quote the pattern using Pattern.quote("regex") for it to parse correctly through both the selector parser and the regex parser. E.g. String query = "div:matches(" + Pattern.quote(regex) + ");".
Escaping special characters: to match a tag, ID, or other selector that does not follow the regular CSS syntax, the query must be escaped with the \ character. For example, to match by ID {@code
}, use {@code document.select("#i\\.d")}.
@see Element#select(String css)
@see Element#selectFirst(String css)
@see Element#select(Evaluator eval)
@see Element#selectNodes(String css)
@see Element#selectNodes(String css, Class nodeType)
@see Elements#select(String css)
@see Element#selectXpath(String xpath) */
public class Selector {
// not instantiable
private Selector() {}
/**
Find Elements matching the CSS query.
@param query CSS selector
@param root root element to descend into
@return matching elements, empty if none
@throws Selector.SelectorParseException (unchecked) on an invalid CSS query.
*/
public static Elements select(String query, Element root) {
Validate.notEmpty(query);
return select(evaluatorOf(query), root);
}
/**
Find Elements matching the Evaluator.
@param evaluator CSS Evaluator
@param root root (context) element to start from
@return matching elements, empty if none
*/
public static Elements select(Evaluator evaluator, Element root) {
Validate.notNull(evaluator);
Validate.notNull(root);
return Collector.collect(evaluator, root);
}
/**
Finds a Stream of elements matching the CSS query.
@param query CSS selector
@param root root element to descend into
@return a Stream of matching elements, empty if none
@throws Selector.SelectorParseException (unchecked) on an invalid CSS query.
@since 1.19.1
*/
public static Stream selectStream(String query, Element root) {
Validate.notEmpty(query);
return selectStream(evaluatorOf(query), root);
}
/**
Finds a Stream of elements matching the evaluator.
@param evaluator CSS selector
@param root root element to descend into
@return matching elements, empty if none
@since 1.19.1
*/
public static Stream selectStream(Evaluator evaluator, Element root) {
Validate.notNull(evaluator);
Validate.notNull(root);
return Collector.stream(evaluator, root);
}
/**
Find elements matching the query, across multiple roots. Elements will be deduplicated (in the case of
overlapping hierarchies).
@param query CSS selector
@param roots root elements to descend into
@return matching elements, empty if none
*/
public static Elements select(String query, Iterable roots) {
Validate.notEmpty(query);
Validate.notNull(roots);
Evaluator evaluator = evaluatorOf(query);
Elements elements = new Elements();
HashSet seenElements = new HashSet<>(); // dedupe elements by identity, as .equals is ==
for (Element root : roots) {
selectStream(evaluator, root)
.filter(seenElements::add)
.forEach(elements::add);
}
return elements;
}
// exclude set. package open so that Elements can implement .not() selector.
static Elements filterOut(Collection elements, Collection outs) {
Elements output = new Elements();
for (Element el : elements) {
boolean found = false;
for (Element out : outs) {
if (el.equals(out)) {
found = true;
break;
}
}
if (!found)
output.add(el);
}
return output;
}
/**
Find the first Element that matches the query.
@param cssQuery CSS selector
@param root root element to descend into
@return the matching element, or null if none.
*/
public static @Nullable Element selectFirst(String cssQuery, Element root) {
Validate.notEmpty(cssQuery);
return Collector.findFirst(evaluatorOf(cssQuery), root);
}
/**
Find the first element matching the query, across multiple roots.
@param cssQuery CSS selector
@param roots root elements to descend into
@return the first matching element, or {@code null} if none
@since 1.19.1
*/
public static @Nullable Element selectFirst(String cssQuery, Iterable roots) {
Validate.notEmpty(cssQuery);
Validate.notNull(roots);
Evaluator evaluator = evaluatorOf(cssQuery);
for (Element root : roots) {
Element first = Collector.findFirst(evaluator, root);
if (first != null) return first;
}
return null;
}
/**
Given a CSS identifier (such as a tag, ID, or class), escape any CSS special characters that would otherwise not be
valid in a selector.
@see CSS Object Model, serialize an identifier
@since 1.20.1
*/
public static String escapeCssIdentifier(String in) {
return TokenQueue.escapeCssIdentifier(in);
}
/**
Consume a CSS identifier (ID or class) off the queue.
Note: For backwards compatibility this method supports improperly formatted CSS identifiers, e.g. {@code 1} instead
of {@code \31}.
@return The unescaped identifier.
@throws IllegalArgumentException if an invalid escape sequence was found.
@see CSS Syntax Module Level 3, Consume an ident sequence
@see CSS Syntax Module Level 3, ident-token
@since 1.20.1
*/
public static String unescapeCssIdentifier(String in) {
try (TokenQueue tq = new TokenQueue(in)) {
return tq.consumeCssIdentifier();
}
}
/**
Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to
parse it once and reuse the Evaluator.
@param css CSS query
@return Evaluator
@see Selector selector query syntax
@throws Selector.SelectorParseException if the CSS query is invalid
@since 1.21.1
*/
public static Evaluator evaluatorOf(String css) {
return QueryParser.parse(css);
}
public static class SelectorParseException extends IllegalStateException {
public SelectorParseException(String msg) {
super(msg);
}
public SelectorParseException(String msg, Object... msgArgs) {
super(String.format(msg, msgArgs));
}
public SelectorParseException(Throwable cause, String msg, Object... msgArgs) {
super(String.format(msg, msgArgs), cause);
}
}
}
================================================
FILE: src/main/java/org/jsoup/select/StructuralEvaluator.java
================================================
package org.jsoup.select;
import org.jsoup.internal.SoftPool;
import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.LeafNode;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.NodeIterator;
import org.jsoup.nodes.TextNode;
import java.util.ArrayList;
import java.util.Map;
import java.util.WeakHashMap;
/**
* Base structural evaluator.
*/
abstract class StructuralEvaluator extends Evaluator {
final Evaluator evaluator;
boolean wantsNodes; // if the evaluator requested nodes, not just elements
public StructuralEvaluator(Evaluator evaluator) {
this.evaluator = evaluator;
wantsNodes = evaluator.wantsNodes();
}
@Override
boolean wantsNodes() {
return wantsNodes;
}
// Memoize inner matches, to save repeated re-evaluations of parent, sibling etc.
// root + element: Boolean matches. ThreadLocal in case the Evaluator is compiled then reused across multi threads
final ThreadLocal>> threadMemo = ThreadLocal.withInitial(WeakHashMap::new);
boolean memoMatches(final Element root, final Node node) {
Map> rootMemo = threadMemo.get();
Map memo = rootMemo.computeIfAbsent(root, r -> new WeakHashMap<>());
return memo.computeIfAbsent(node, test -> evaluator.matches(root, test));
}
@Override protected void reset() {
threadMemo.remove();
evaluator.reset();
super.reset();
}
@Override
public boolean matches(Element root, Element element) {
return evaluateMatch(root, element);
}
@Override
boolean matches(Element root, LeafNode leafNode) {
return evaluateMatch(root, leafNode);
}
abstract boolean evaluateMatch(Element root, Node node);
static class Root extends Evaluator {
@Override
public boolean matches(Element root, Element element) {
return root == element;
}
@Override protected int cost() {
return 1;
}
@Override public String toString() {
return ">";
}
}
static class Has extends StructuralEvaluator {
static final SoftPool> NodeIterPool =
new SoftPool<>(() -> new NodeIterator<>(new TextNode(""), Node.class));
// the element here is just a placeholder so this can be final - gets set in restart()
private final boolean checkSiblings; // evaluating against siblings (or children)
public Has(Evaluator evaluator) {
super(evaluator);
checkSiblings = evalWantsSiblings(evaluator);
}
@Override public boolean matches(Element root, Element element) {
if (checkSiblings) { // evaluating against siblings
for (Element sib = element.firstElementSibling(); sib != null; sib = sib.nextElementSibling()) {
if (sib != element && evaluator.matches(element, sib)) { // don't match against self
return true;
}
}
}
// otherwise we only want to match children (or below), and not the input element. And we want to minimize GCs so reusing the Iterator obj
NodeIterator it = NodeIterPool.borrow();
it.restart(element);
try {
while (it.hasNext()) {
Node node = it.next();
if (node == element) continue; // don't match self, only descendants
if (evaluator.matches(element, node)) {
return true;
}
}
} finally {
NodeIterPool.release(it);
}
return false;
}
@Override
boolean evaluateMatch(Element root, Node node) {
return false; // unused; :has(::comment)) goes via implicit root combinator
}
/* Test if the :has sub-clause wants sibling elements (vs nested elements) - will be a Combining eval */
private static boolean evalWantsSiblings(Evaluator eval) {
if (eval instanceof CombiningEvaluator) {
CombiningEvaluator ce = (CombiningEvaluator) eval;
for (Evaluator innerEval : ce.evaluators) {
if (innerEval instanceof PreviousSibling || innerEval instanceof ImmediatePreviousSibling)
return true;
}
}
return false;
}
@Override protected int cost() {
return 10 * evaluator.cost();
}
@Override
public String toString() {
return String.format(":has(%s)", evaluator);
}
}
/** Implements the :is(sub-query) pseudo-selector */
static class Is extends StructuralEvaluator {
public Is(Evaluator evaluator) {
super(evaluator);
}
@Override
boolean evaluateMatch(Element root, Node node) {
return evaluator.matches(root, node);
}
@Override protected int cost() {
return 2 + evaluator.cost();
}
@Override
public String toString() {
return String.format(":is(%s)", evaluator);
}
}
static class Not extends StructuralEvaluator {
public Not(Evaluator evaluator) {
super(evaluator);
}
@Override
boolean evaluateMatch(Element root, Node node) {
return !memoMatches(root, node);
}
@Override protected int cost() {
return 2 + evaluator.cost();
}
@Override
public String toString() {
return String.format(":not(%s)", evaluator);
}
}
/**
Any Ancestor (i.e., ascending parent chain.).
*/
static class Ancestor extends StructuralEvaluator {
public Ancestor(Evaluator evaluator) {
super(evaluator);
}
@Override
boolean evaluateMatch(Element root, Node node) {
if (root == node)
return false;
for (Node parent = node.parent(); parent != null; parent = parent.parent()) {
if (memoMatches(root, parent))
return true;
if (parent == root)
break;
}
return false;
}
@Override
protected int cost() {
return 8 * evaluator.cost(); // probably lower than has(), but still significant, depending on doc and el depth.
}
@Override
public String toString() {
return String.format("%s ", evaluator);
}
}
/**
Holds a list of evaluators for one > two > three immediate parent matches, and the final direct evaluator under
test. To match, these are effectively ANDed together, starting from the last, matching up to the first.
*/
static class ImmediateParentRun extends StructuralEvaluator {
final ArrayList evaluators = new ArrayList<>();
int cost = 2;
public ImmediateParentRun(Evaluator evaluator) {
super(evaluator);
evaluators.add(evaluator);
cost += evaluator.cost();
}
void add(Evaluator evaluator) {
evaluators.add(evaluator);
cost += evaluator.cost();
wantsNodes |= evaluator.wantsNodes();
}
@Override boolean evaluateMatch(Element root, Node node) {
if (node == root)
return false; // cannot match as the second eval (first parent test) would be above the root
for (int i = evaluators.size() -1; i >= 0; --i) {
if (node == null)
return false;
Evaluator eval = evaluators.get(i);
if (!eval.matches(root, node))
return false;
node = node.parent();
}
return true;
}
@Override protected int cost() {
return cost;
}
@Override
protected void reset() {
for (Evaluator evaluator : evaluators) {
evaluator.reset();
}
super.reset();
}
@Override
public String toString() {
return StringUtil.join(evaluators, " > ");
}
}
static class PreviousSibling extends StructuralEvaluator {
public PreviousSibling(Evaluator evaluator) {
super(evaluator);
}
// matches any previous sibling, so can be same in Element only or wantsNodes context
@Override boolean evaluateMatch(Element root, Node node) {
if (root == node) return false;
for (Node sib = node.firstSibling(); sib != null; sib = sib.nextSibling()) {
if (sib == node) break;
if (memoMatches(root, sib)) return true;
}
return false;
}
@Override protected int cost() {
return 3 * evaluator.cost();
}
@Override
public String toString() {
return String.format("%s ~ ", evaluator);
}
}
static class ImmediatePreviousSibling extends StructuralEvaluator {
public ImmediatePreviousSibling(Evaluator evaluator) {
super(evaluator);
}
@Override boolean evaluateMatch(Element root, Node node) {
if (root == node) return false;
Node prev = wantsNodes ? node.previousSibling() : node.previousElementSibling();
return prev != null && memoMatches(root, prev);
}
@Override protected int cost() {
return 2 + evaluator.cost();
}
@Override
public String toString() {
return String.format("%s + ", evaluator);
}
}
}
================================================
FILE: src/main/java/org/jsoup/select/package-info.java
================================================
/**
Packages to support the CSS-style element selector.
{@link org.jsoup.select.Selector Selector defines the query syntax.}
*/
@NullMarked
package org.jsoup.select;
import org.jspecify.annotations.NullMarked;
================================================
FILE: src/main/java11/module-info.java
================================================
module org.jsoup {
exports org.jsoup;
exports org.jsoup.helper;
exports org.jsoup.nodes;
exports org.jsoup.parser;
exports org.jsoup.safety;
exports org.jsoup.select;
requires transitive java.xml; // for org.w3c.dom out of W3CDom
requires static org.jspecify; // nullability annotations
requires static java.net.http; // HttpClient on Java 11; guarded
}
================================================
FILE: src/main/java11/org/jsoup/helper/HttpClientExecutor.java
================================================
package org.jsoup.helper;
import org.jsoup.Connection;
import org.jspecify.annotations.Nullable;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpHeaders;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.jsoup.helper.HttpConnection.Response;
import static org.jsoup.helper.HttpConnection.Response.writePost;
/**
Executes requests using the HttpClient, for http/2 support. Enabled by default when available. To disable, set
property {@code jsoup.useHttpClient} to {@code false}.
*/
class HttpClientExecutor extends RequestExecutor {
// HttpClient expects proxy settings per client; we do per request, so held as a thread local. Can't do same for
// auth because that callback is on a worker thread, so can only do auth per Connection. So we create a new client
// if the authenticator is different between requests
static ThreadLocal<@Nullable Proxy> perRequestProxy = new ThreadLocal<>();
@Nullable
HttpResponse hRes;
public HttpClientExecutor(HttpConnection.Request request, HttpConnection.@Nullable Response previousResponse) {
super(request, previousResponse);
}
/**
Retrieve the HttpClient from the Connection, or create a new one. Allows for connection pooling of requests in the
same Connection (session).
*/
HttpClient client() {
// we try to reuse the same Client across requests in a given Connection; but if the request's auth or ssl context have changed, we need to create a new client
if (req.connection.client != null) {
HttpClient client = (HttpClient) req.connection.client;
boolean reuse = true;
RequestAuthenticator prevAuth = req.connection.lastAuth;
req.connection.lastAuth = req.authenticator;
if (prevAuth != req.authenticator) // might both be null
reuse = false;
if (req.sslContext != null && !(client.sslContext() == req.sslContext)) // client returns default context if not otherwise set
reuse = false;
if (reuse) return client;
}
HttpClient.Builder builder = HttpClient.newBuilder();
builder.followRedirects(HttpClient.Redirect.NEVER); // customized redirects
builder.proxy(new ProxyWrap()); // thread local impl for per request; called on executing thread
if (req.authenticator != null) builder.authenticator(new AuthenticationHandler(req.authenticator));
if (req.sslContext != null) builder.sslContext(req.sslContext);
HttpClient client = builder.build();
req.connection.client = client;
return client;
}
@Override
HttpConnection.Response execute() throws IOException {
try {
HttpRequest.Builder reqBuilder =
HttpRequest.newBuilder(req.url.toURI()).method(req.method.name(), requestBody(req));
if (req.timeout() > 0) reqBuilder.timeout(
Duration.ofMillis(req.timeout())); // infinite if unset (UrlConnection / jsoup uses 0 for same)
CookieUtil.applyCookiesToRequest(req, reqBuilder::header);
// headers:
req.multiHeaders().forEach((key, values) -> {
values.forEach(value -> reqBuilder.header(key, value));
});
if (req.proxy() != null) perRequestProxy.set(req.proxy()); // set up per request proxy
HttpRequest hReq = reqBuilder.build();
HttpClient client = client();
hRes = client.send(hReq, HttpResponse.BodyHandlers.ofInputStream());
HttpHeaders headers = hRes.headers();
// set up the response
Response res = new Response(req);
res.executor = this;
res.method = Connection.Method.valueOf(hRes.request().method());
res.url = hRes.uri().toURL();
res.statusCode = hRes.statusCode();
res.statusMessage = StatusMessage(res.statusCode);
res.contentType = headers.firstValue("content-type").orElse("");
long length = headers.firstValueAsLong("content-length").orElse(-1);
res.contentLength = length < Integer.MAX_VALUE ? (int) length : -1;
res.prepareResponse(headers.map(), prevRes);
return res;
} catch (IOException e) {
safeClose();
throw e;
} catch (InterruptedException e) {
safeClose();
Thread.currentThread().interrupt();
throw new IOException(e);
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Malformed URL: " + req.url, e);
} finally {
// detach per request proxy
perRequestProxy.remove();
}
}
/**
As HTTP/2 no longer provides a server-set status message, and HttpClient doesn't parse it for 1.1, just provide minimal stock ones, for loggers.
*/
static String StatusMessage(int statusCode) {
if (statusCode < 400) return "OK";
if (statusCode == 404) return "Not Found";
return "Error " + statusCode;
}
@Override
InputStream responseBody() throws IOException {
if (hRes == null) throw new IllegalStateException("Not yet executed");
return hRes.body();
}
@Override
void safeClose() {
if (hRes != null) {
InputStream body = hRes.body();
if (body != null) {
try {
body.close();
} catch (IOException ignored) {}
}
hRes = null;
}
}
static HttpRequest.BodyPublisher requestBody(final HttpConnection.Request req) throws IOException {
if (req.method.hasBody()) {
ByteArrayOutputStream buf = new ByteArrayOutputStream();
writePost(req, buf);
return HttpRequest.BodyPublishers.ofByteArray(buf.toByteArray());
} else {
return HttpRequest.BodyPublishers.noBody();
}
}
static class ProxyWrap extends ProxySelector {
// empty list for no proxy:
static final List NoProxy = new ArrayList<>(0);
@Override
public List select(URI uri) {
Proxy proxy = perRequestProxy.get();
if (proxy != null) {
return Collections.singletonList(proxy);
}
ProxySelector defaultSelector = ProxySelector.getDefault();
if (defaultSelector != null && defaultSelector != this) { // avoid recursion if we were set as default
return defaultSelector.select(uri);
}
return NoProxy;
}
@Override
public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
if (perRequestProxy.get() != null) {
return; // no-op
}
ProxySelector defaultSelector = ProxySelector.getDefault();
if (defaultSelector != null && defaultSelector != this) {
defaultSelector.connectFailed(uri, sa, ioe);
}
}
}
}
================================================
FILE: src/main/java11/org/jsoup/helper/RequestAuthHandler.java
================================================
package org.jsoup.helper;
import java.net.HttpURLConnection;
import java.net.http.HttpClient;
/**
A per-request authentication shim, used in Java 9+.
*/
class RequestAuthHandler implements AuthenticationHandler.AuthShim {
public RequestAuthHandler() {}
@Override public void enable(RequestAuthenticator auth, Object connOrHttp) {
AuthenticationHandler authenticator = new AuthenticationHandler(auth);
// this is a bit ugly, but a simple way to support setting authentication on both urlconnection and httpclient without more multi-version shims
if (connOrHttp instanceof HttpURLConnection) {
HttpURLConnection conn = (HttpURLConnection) connOrHttp;
conn.setAuthenticator(authenticator);
} else if (connOrHttp instanceof HttpClient.Builder) {
HttpClient.Builder builder = (HttpClient.Builder) connOrHttp;
builder.authenticator(authenticator);
} else {
throw new IllegalArgumentException("Unsupported executor: " + connOrHttp.getClass().getName());
}
}
@Override public void remove() {
// noop; would remove thread-local in Global Handler
}
@Override public AuthenticationHandler get(AuthenticationHandler helper) {
// would get thread-local in Global Handler
return helper;
}
}
================================================
FILE: src/main/javadoc/overview.html
================================================
jsoup Javadoc overview
jsoup: Java HTML parser that makes sense of real-world HTML soup.
jsoup is a Java library for working with real-world HTML. It provides a very convenient API for fetching URLs
and extracting and manipulating data, using the best of HTML5 DOM methods and CSS selectors.
jsoup implements the WHATWG HTML specification, and parses HTML to the same DOM
as modern browsers do.
- parse HTML from a URL, file, or string
- find and extract data, using DOM traversal or CSS selectors
- manipulate the HTML elements, attributes, and text
- clean user-submitted content against a safelist, to prevent XSS
- output tidy HTML
jsoup is designed to deal with all varieties of HTML found in the wild; from pristine and validating,
to invalid tag-soup; jsoup will create a sensible parse tree.
See jsoup.org for downloads, documentation, and examples.
@author Jonathan Hedley
================================================
FILE: src/main/resources/META-INF/proguard/org.jsoup_jsoup.pro
================================================
-dontwarn com.google.re2j.**
================================================
FILE: src/test/java/org/jsoup/JsoupTest.java
================================================
package org.jsoup;
import org.jsoup.integration.ParseTest;
import org.jsoup.nodes.Document;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class JsoupTest {
// Tests for the Jsoup class. Mostly for code coverage for methods that haven't been covered elsewhere already.
@Test
void parseWithPath() throws IOException {
// parse(Path path, @Nullable String charsetName, String baseUri)
Path path = ParseTest.getPath("/htmltests/medium.html");
Document doc = Jsoup.parse(path, "UTF-8", "https://example.com/");
String title = "Medium HTML";
assertEquals(title, doc.title());
// parse(Path path)
doc = Jsoup.parse(path);
assertEquals(title, doc.title());
// (Path path, @Nullable String charsetName, String baseUri, Parser parser)
doc = Jsoup.parse(path, "UTF-8", "https://example.com/", Parser.htmlParser());
assertEquals(title, doc.title());
}
}
================================================
FILE: src/test/java/org/jsoup/MultiLocaleExtension.java
================================================
package org.jsoup;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.ArgumentsProvider;
import org.junit.jupiter.params.provider.ArgumentsSource;
import java.lang.annotation.*;
import java.util.Locale;
import java.util.stream.Stream;
public class MultiLocaleExtension implements AfterEachCallback, ArgumentsProvider {
private final Locale defaultLocale = Locale.getDefault();
@Override
public void afterEach(ExtensionContext context) {
Locale.setDefault(defaultLocale);
}
@Override
public Stream provideArguments(ExtensionContext extensionContext) {
return Stream.of(Arguments.of(Locale.ENGLISH), Arguments.arguments(new Locale("tr")));
}
@Documented
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@ArgumentsSource(MultiLocaleExtension.class)
@ExtendWith(MultiLocaleExtension.class)
@ParameterizedTest
public @interface MultiLocaleTest {
}
}
================================================
FILE: src/test/java/org/jsoup/SerializationExceptionTest.java
================================================
package org.jsoup;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
public class SerializationExceptionTest {
@Test void constructors() {
SerializationException e = new SerializationException("message");
assertEquals("message", e.getMessage());
assertNull(e.getCause());
SerializationException e2 = new SerializationException("message", new Exception("cause"));
assertEquals("message", e2.getMessage());
assertEquals("cause", e2.getCause().getMessage());
SerializationException e3 = new SerializationException();
assertNull(e3.getMessage());
assertNull(e3.getCause());
}
}
================================================
FILE: src/test/java/org/jsoup/TextUtil.java
================================================
package org.jsoup;
import java.util.regex.Pattern;
/**
Text utils to ease testing
@author Jonathan Hedley, jonathan@hedley.net */
public class TextUtil {
static Pattern stripper = Pattern.compile("\\r?\\n\\s*");
static Pattern stripLines = Pattern.compile("\\r?\\n?");
static Pattern spaceCollapse = Pattern.compile("\\s{2,}");
static Pattern tagSpaceCollapse = Pattern.compile(">\\s+<");
static Pattern stripCRs = Pattern.compile("\\r*");
public static String stripNewlines(String text) {
return stripper.matcher(text).replaceAll("");
}
public static String normalizeSpaces(String text) {
text = stripLines.matcher(text).replaceAll("");
text = stripper.matcher(text).replaceAll("");
text = spaceCollapse.matcher(text).replaceAll(" ");
text = tagSpaceCollapse.matcher(text).replaceAll("><");
return text;
}
public static String stripCRs(String text) {
return stripCRs.matcher(text).replaceAll("");
}
}
================================================
FILE: src/test/java/org/jsoup/helper/AuthenticationHandlerTest.java
================================================
package org.jsoup.helper;
public class AuthenticationHandlerTest {
public static final int MaxAttempts = AuthenticationHandler.MaxAttempts;
// tests are in ConnectionTest, ProxyTest. This class just makes the MaxAttempts visible for test.
}
================================================
FILE: src/test/java/org/jsoup/helper/CookieUtilTest.java
================================================
package org.jsoup.helper;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class CookieUtilTest {
@Test void parseCookie() {
HttpConnection.Response res = new HttpConnection.Response();
CookieUtil.parseCookie("foo=bar qux; Domain=.example.com; Path=/; Secure", res);
CookieUtil.parseCookie("bar=foo qux", res);
CookieUtil.parseCookie("=bar; Domain=.example.com; Path=/; Secure", res);
CookieUtil.parseCookie("; Domain=.example.com; Path=/", res);
CookieUtil.parseCookie("", res);
CookieUtil.parseCookie(null, res);
assertEquals(3, res.cookies().size());
assertEquals("bar qux", res.cookies.get("foo"));
assertEquals("foo qux", res.cookies.get("bar"));
assertEquals(".example.com", res.cookies.get("; Domain")); // no actual cookie name or val
}
}
================================================
FILE: src/test/java/org/jsoup/helper/DataUtilTest.java
================================================
package org.jsoup.helper;
import org.jsoup.Jsoup;
import org.jsoup.integration.ParseTest;
import org.jsoup.internal.ControllableInputStream;
import org.jsoup.nodes.Document;
import org.jsoup.parser.Parser;
import org.jsoup.parser.StreamParser;
import org.junit.jupiter.api.Test;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.jsoup.integration.ParseTest.getFile;
import static org.jsoup.integration.ParseTest.getPath;
import static org.junit.jupiter.api.Assertions.*;
public class DataUtilTest {
@Test
public void testCharset() {
assertEquals("utf-8", DataUtil.getCharsetFromContentType("text/html;charset=utf-8 "));
assertEquals("UTF-8", DataUtil.getCharsetFromContentType("text/html; charset=UTF-8"));
assertEquals("ISO-8859-1", DataUtil.getCharsetFromContentType("text/html; charset=ISO-8859-1"));
assertNull(DataUtil.getCharsetFromContentType("text/html"));
assertNull(DataUtil.getCharsetFromContentType(null));
assertNull(DataUtil.getCharsetFromContentType("text/html;charset=Unknown"));
}
@Test
public void testQuotedCharset() {
assertEquals("utf-8", DataUtil.getCharsetFromContentType("text/html; charset=\"utf-8\""));
assertEquals("UTF-8", DataUtil.getCharsetFromContentType("text/html;charset=\"UTF-8\""));
assertEquals("ISO-8859-1", DataUtil.getCharsetFromContentType("text/html; charset=\"ISO-8859-1\""));
assertNull(DataUtil.getCharsetFromContentType("text/html; charset=\"Unsupported\""));
assertEquals("UTF-8", DataUtil.getCharsetFromContentType("text/html; charset='UTF-8'"));
}
private ControllableInputStream stream(String data) {
return ControllableInputStream.wrap(new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)), 0);
}
private ControllableInputStream stream(String data, String charset) {
return ControllableInputStream.wrap(new ByteArrayInputStream(data.getBytes(Charset.forName(charset))), 0);
}
@Test
public void discardsSpuriousByteOrderMark() throws IOException {
String html = "\uFEFFOneTwo";
Document doc = DataUtil.parseInputStream(stream(html), "UTF-8", "http://foo.com/", Parser.htmlParser());
assertEquals("One", doc.head().text());
}
@Test
public void discardsSpuriousByteOrderMarkWhenNoCharsetSet() throws IOException {
String html = "\uFEFFOneTwo";
Document doc = DataUtil.parseInputStream(stream(html), null, "http://foo.com/", Parser.htmlParser());
assertEquals("One", doc.head().text());
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
}
@Test
public void shouldNotThrowExceptionOnEmptyCharset() {
assertNull(DataUtil.getCharsetFromContentType("text/html; charset="));
assertNull(DataUtil.getCharsetFromContentType("text/html; charset=;"));
}
@Test
public void shouldSelectFirstCharsetOnWeirdMultileCharsetsInMetaTags() {
assertEquals("ISO-8859-1", DataUtil.getCharsetFromContentType("text/html; charset=ISO-8859-1, charset=1251"));
}
@Test
public void shouldCorrectCharsetForDuplicateCharsetString() {
assertEquals("iso-8859-1", DataUtil.getCharsetFromContentType("text/html; charset=charset=iso-8859-1"));
}
@Test
public void shouldReturnNullForIllegalCharsetNames() {
assertNull(DataUtil.getCharsetFromContentType("text/html; charset=$HJKDF§$/("));
}
@Test
public void generatesMimeBoundaries() {
String m1 = DataUtil.mimeBoundary();
String m2 = DataUtil.mimeBoundary();
assertEquals(DataUtil.boundaryLength, m1.length());
assertEquals(DataUtil.boundaryLength, m2.length());
assertNotSame(m1, m2);
}
@Test
public void wrongMetaCharsetFallback() throws IOException {
String html = "";
Document doc = DataUtil.parseInputStream(stream(html), null, "http://example.com", Parser.htmlParser());
final String expected = "\n" +
" \n" +
" \n" +
" \n" +
" \n" +
"";
assertEquals(expected, doc.toString());
}
@Test
public void secondMetaElementWithContentTypeContainsCharsetParameter() throws Exception {
String html = "" +
"" +
"" +
"한국어";
Document doc = DataUtil.parseInputStream(stream(html, "euc-kr"), null, "http://example.com", Parser.htmlParser());
assertEquals("한국어", doc.body().text());
}
@Test
public void firstMetaElementWithCharsetShouldBeUsedForDecoding() throws Exception {
String html = "" +
"" +
"" +
"Übergrößenträger";
Document doc = DataUtil.parseInputStream(stream(html, "iso-8859-1"), null, "http://example.com", Parser.htmlParser());
assertEquals("Übergrößenträger", doc.body().text());
}
@Test
public void parseSequenceInputStream() throws IOException {
// https://github.com/jhy/jsoup/pull/1671
File in = getFile("/htmltests/medium.html");
String fileContent = new String(Files.readAllBytes(in.toPath()));
int halfLength = fileContent.length() / 2;
String firstPart = fileContent.substring(0, halfLength);
String secondPart = fileContent.substring(halfLength);
SequenceInputStream sequenceStream = new SequenceInputStream(
stream(firstPart),
stream(secondPart)
);
ControllableInputStream stream = ControllableInputStream.wrap(sequenceStream, 0);
Document doc = DataUtil.parseInputStream(stream, null, "", Parser.htmlParser());
assertEquals(fileContent, doc.outerHtml());
}
@Test
public void supportsBOMinFiles() throws IOException {
// test files from http://www.i18nl10n.com/korean/utftest/
File in = getFile("/bomtests/bom_utf16be.html");
Document doc = Jsoup.parse(in, null, "http://example.com");
assertTrue(doc.title().contains("UTF-16BE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf16le.html");
doc = Jsoup.parse(in, null, "http://example.com");
assertTrue(doc.title().contains("UTF-16LE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf32be.html");
doc = Jsoup.parse(in, null, "http://example.com");
assertTrue(doc.title().contains("UTF-32BE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf32le.html");
doc = Jsoup.parse(in, null, "http://example.com");
assertTrue(doc.title().contains("UTF-32LE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
}
@Test
public void streamerSupportsBOMinFiles() throws IOException {
// test files from http://www.i18nl10n.com/korean/utftest/
Path in = getFile("/bomtests/bom_utf16be.html").toPath();
Parser parser = Parser.htmlParser();
Document doc = DataUtil.streamParser(in, null, "http://example.com", parser).complete();
assertTrue(doc.title().contains("UTF-16BE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf16le.html").toPath();
doc = DataUtil.streamParser(in, null, "http://example.com", parser).complete();
assertTrue(doc.title().contains("UTF-16LE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf32be.html").toPath();
doc = DataUtil.streamParser(in, null, "http://example.com", parser).complete();
assertTrue(doc.title().contains("UTF-32BE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
in = getFile("/bomtests/bom_utf32le.html").toPath();
doc = DataUtil.streamParser(in, null, "http://example.com", parser).complete();
assertTrue(doc.title().contains("UTF-32LE"));
assertTrue(doc.text().contains("가각갂갃간갅"));
}
@Test
public void supportsUTF8BOM() throws IOException {
File in = getFile("/bomtests/bom_utf8.html");
Document doc = Jsoup.parse(in, null, "http://example.com");
assertEquals("OK", doc.head().select("title").text());
}
@Test
public void noExtraNULLBytes() throws IOException {
final byte[] b = "üü
".getBytes(StandardCharsets.UTF_8);
Document doc = Jsoup.parse(new ByteArrayInputStream(b), null, "");
assertFalse( doc.outerHtml().contains("\u0000") );
}
@Test
public void supportsZippedUTF8BOM() throws IOException {
File in = getFile("/bomtests/bom_utf8.html.gz");
Document doc = Jsoup.parse(in, null, "http://example.com");
assertEquals("OK", doc.head().select("title").text());
assertEquals("There is a UTF8 BOM at the top (before the XML decl). If not read correctly, will look like a non-joining space.", doc.body().text());
}
@Test
public void streamerSupportsZippedUTF8BOM() throws IOException {
Path in = getFile("/bomtests/bom_utf8.html.gz").toPath();
Document doc = DataUtil.streamParser(in, null, "http://example.com", Parser.htmlParser()).complete();
assertEquals("OK", doc.head().select("title").text());
assertEquals("There is a UTF8 BOM at the top (before the XML decl). If not read correctly, will look like a non-joining space.", doc.body().text());
}
@Test
public void supportsXmlCharsetDeclaration() throws IOException {
String encoding = "iso-8859-1";
InputStream soup = new ByteArrayInputStream((
"" +
"" +
"Hellö Wörld!"
).getBytes(Charset.forName(encoding)));
Document doc = Jsoup.parse(soup, null, "");
assertEquals("Hellö Wörld!", doc.body().text());
}
@Test
public void loadsGzipFile() throws IOException {
File in = getFile("/htmltests/gzip.html.gz");
Document doc = Jsoup.parse(in, null);
assertEquals("Gzip test", doc.title());
assertEquals("This is a gzipped HTML file.", doc.selectFirst("p").text());
}
@Test
public void loadsGzipPath() throws IOException {
Path in = getPath("/htmltests/gzip.html.gz");
Document doc = Jsoup.parse(in, null);
assertEquals("Gzip test", doc.title());
assertEquals("This is a gzipped HTML file.", doc.selectFirst("p").text());
}
@Test
public void loadsZGzipFile() throws IOException {
// compressed on win, with z suffix
File in = getFile("/htmltests/gzip.html.z");
Document doc = Jsoup.parse(in, null);
assertEquals("Gzip test", doc.title());
assertEquals("This is a gzipped HTML file.", doc.selectFirst("p").text());
}
@Test
public void loadsZGzipPath() throws IOException {
// compressed on win, with z suffix
Path in = getPath("/htmltests/gzip.html.z");
Document doc = Jsoup.parse(in, null);
assertEquals("Gzip test", doc.title());
assertEquals("This is a gzipped HTML file.", doc.selectFirst("p").text());
}
@Test
public void handlesFakeGzipFile() throws IOException {
File in = getFile("/htmltests/fake-gzip.html.gz");
Document doc = Jsoup.parse(in, null);
assertEquals("This is not gzipped", doc.title());
assertEquals("And should still be readable.", doc.selectFirst("p").text());
}
@Test
public void handlesFakeGzipPath() throws IOException {
Path in = getPath("/htmltests/fake-gzip.html.gz");
Document doc = Jsoup.parse(in, null);
assertEquals("This is not gzipped", doc.title());
assertEquals("And should still be readable.", doc.selectFirst("p").text());
}
// an input stream to give a range of output sizes, that changes on each read
static class VaryingReadInputStream extends InputStream {
final InputStream in;
int stride = 0;
VaryingReadInputStream(InputStream in) {
this.in = in;
}
public int read() throws IOException {
return in.read();
}
public int read(byte[] b) throws IOException {
return in.read(b, 0, Math.min(b.length, ++stride));
}
public int read(byte[] b, int off, int len) throws IOException {
return in.read(b, off, Math.min(len, ++stride));
}
}
@Test
void handlesChunkedInputStream() throws IOException {
File inputFile = ParseTest.getFile("/htmltests/large.html");
String input = ParseTest.getFileAsString(inputFile);
VaryingReadInputStream stream = new VaryingReadInputStream(ParseTest.inputStreamFrom(input));
Document expected = Jsoup.parse(input, "https://example.com");
Document doc = Jsoup.parse(stream, null, "https://example.com");
assertTrue(doc.hasSameValue(expected));
}
@Test
void handlesUnlimitedRead() throws IOException {
File inputFile = ParseTest.getFile("/htmltests/large.html");
String input = ParseTest.getFileAsString(inputFile);
VaryingReadInputStream stream = new VaryingReadInputStream(ParseTest.inputStreamFrom(input));
ByteBuffer byteBuffer = DataUtil.readToByteBuffer(stream, 0);
String read = new String(byteBuffer.array(), 0, byteBuffer.limit(), StandardCharsets.UTF_8);
assertEquals(input, read);
}
@Test void controllableInputStreamAllowsNull() throws IOException {
ControllableInputStream is = ControllableInputStream.wrap(null, 0);
assertNotNull(is);
assertTrue(is.baseReadFully());
is.close();
}
@Test void streamParserSurrogateAcrossBuffer() throws IOException {
// https://github.com/jhy/jsoup/issues/2353
try (StreamParser parser = DataUtil.streamParser(ParseTest.getPath("/fuzztests/2353.html.gz"), DataUtil.UTF_8, "", Parser.htmlParser())) {
Document doc = parser.complete();
String html = doc.html();
assertTrue(html.contains("Read-Fully!"));
}
}
@Test void parseSurrogateAcrossBuffer() throws IOException {
Document doc = Jsoup.parse(ParseTest.getPath("/fuzztests/2353.html.gz"));
assertTrue(doc.html().contains("Read-Fully!"));
}
@Test
void charsetSniffingCanReuseTruncatedPreParse() throws IOException {
// #2448: when available() reports buffered bytes after the first read, the sniffed pre-parse may be reused while capped, leading to truncation
StringBuilder sb = new StringBuilder();
sb.append("t");
while (sb.length() < 6200) {
sb.append("0123456789 abcdefghijklmnopqrstuvwxyz\n");
}
sb.append("list
");
String html = sb.toString();
byte[] bytes = html.getBytes(StandardCharsets.UTF_8);
ControllableInputStream in = ControllableInputStream.wrap(new BufferedOnceAvailableStream(bytes), 0);
DataUtil.CharsetDoc charsetDoc = DataUtil.detectCharset(in, null, "http://example.com/", Parser.htmlParser());
Document doc = DataUtil.parseInputStream(charsetDoc, "http://example.com/", Parser.htmlParser());
assertNotNull(doc.selectFirst("hr"), "hr should survive the sniff + full parse");
}
// delivers all bytes in the first read, then signals available()>0 once to trigger a second read and baseReadFully=true
static final class BufferedOnceAvailableStream extends InputStream {
private final byte[] data;
private int pos = 0;
private boolean extraSignal = true;
BufferedOnceAvailableStream(byte[] data) {
this.data = data;
}
@Override
public int read(byte[] b, int off, int len) {
if (pos >= data.length) return -1;
int take = Math.min(len, data.length - pos);
System.arraycopy(data, pos, b, off, take);
pos += take;
return take;
}
@Override
public int read() {
return pos < data.length ? (data[pos++] & 0xff) : -1;
}
@Override
public int available() {
if (pos < data.length)
return data.length - pos;
if (extraSignal) {
extraSignal = false;
return 1; // nudge SimpleBufferedInput.fill() to try another read
}
return 0;
}
}
@Test
void charsetSniffingIgnoresAdvisoryAvailableIOException() throws IOException {
// https://github.com/jhy/jsoup/issues/2474
// JDK 8's HttpURLConnection stream may throw from available() once the peer has closed the socket;
// that advisory failure does not mean we can't still consume bytes already buffered or read to clean EOF.
String html = "OneTwo";
byte[] bytes = html.getBytes(StandardCharsets.UTF_8);
InputStream stream = new FilterInputStream(new ByteArrayInputStream(bytes)) {
@Override
public int available() throws IOException {
throw new IOException("Stream closed.");
}
};
ControllableInputStream in = ControllableInputStream.wrap(stream, 0);
Document doc = DataUtil.parseInputStream(in, null, "http://example.com/", Parser.htmlParser());
assertEquals("One", doc.title());
assertEquals("Two", doc.body().text());
}
}
================================================
FILE: src/test/java/org/jsoup/helper/HttpConnectionTest.java
================================================
package org.jsoup.helper;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.MultiLocaleExtension.MultiLocaleTest;
import org.jsoup.integration.ParseTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.net.Authenticator;
import java.net.MalformedURLException;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.jsoup.helper.HttpConnection.Response.fixHeaderEncoding;
import static org.junit.jupiter.api.Assertions.*;
public class HttpConnectionTest {
/* most actual network http connection tests are in integration */
@Test public void canCreateEmptyConnection() {
HttpConnection con = new HttpConnection();
assertEquals(Connection.Method.GET, con.request().method());
assertThrows(IllegalArgumentException.class, () -> {
URL url = con.request().url();
});
}
@Test public void throwsExceptionOnResponseWithoutExecute() {
assertThrows(IllegalArgumentException.class, () -> {
Connection con = HttpConnection.connect("http://example.com");
con.response();
});
}
@Test public void throwsExceptionOnParseWithoutExecute() {
assertThrows(IllegalArgumentException.class, () -> {
Connection con = HttpConnection.connect("http://example.com");
con.response().parse();
});
}
@Test public void throwsExceptionOnBodyWithoutExecute() {
assertThrows(IllegalArgumentException.class, () -> {
Connection con = HttpConnection.connect("http://example.com");
con.response().body();
});
}
@Test public void throwsExceptionOnBodyAsBytesWithoutExecute() {
assertThrows(IllegalArgumentException.class, () -> {
Connection con = HttpConnection.connect("http://example.com");
con.response().bodyAsBytes();
});
}
@MultiLocaleTest
public void caseInsensitiveHeaders(Locale locale) {
Locale.setDefault(locale);
Connection.Response res = new HttpConnection.Response();
res.header("Accept-Encoding", "gzip");
res.header("content-type", "text/html");
res.header("refErrer", "http://example.com");
assertTrue(res.hasHeader("Accept-Encoding"));
assertTrue(res.hasHeader("accept-encoding"));
assertTrue(res.hasHeader("accept-Encoding"));
assertTrue(res.hasHeader("ACCEPT-ENCODING"));
assertEquals("gzip", res.header("accept-Encoding"));
assertEquals("gzip", res.header("ACCEPT-ENCODING"));
assertEquals("text/html", res.header("Content-Type"));
assertEquals("http://example.com", res.header("Referrer"));
res.removeHeader("Content-Type");
assertFalse(res.hasHeader("content-type"));
res.removeHeader("ACCEPT-ENCODING");
assertFalse(res.hasHeader("Accept-Encoding"));
res.header("ACCEPT-ENCODING", "deflate");
assertEquals("deflate", res.header("Accept-Encoding"));
assertEquals("deflate", res.header("accept-Encoding"));
}
@Test public void headers() {
Connection con = HttpConnection.connect("http://example.com");
Map headers = new HashMap<>();
headers.put("content-type", "text/html");
headers.put("Connection", "keep-alive");
headers.put("Host", "http://example.com");
con.headers(headers);
assertEquals("text/html", con.request().header("content-type"));
assertEquals("keep-alive", con.request().header("Connection"));
assertEquals("http://example.com", con.request().header("Host"));
}
@Test public void sameHeadersCombineWithComma() {
Map> headers = new HashMap<>();
List values = new ArrayList<>();
values.add("no-cache");
values.add("no-store");
headers.put("Cache-Control", values);
HttpConnection.Response res = new HttpConnection.Response();
res.processResponseHeaders(headers);
assertEquals("no-cache, no-store", res.header("Cache-Control"));
}
@Test public void multipleHeaders() {
Connection.Request req = new HttpConnection.Request();
req.addHeader("Accept", "Something");
req.addHeader("Accept", "Everything");
req.addHeader("Foo", "Bar");
assertTrue(req.hasHeader("Accept"));
assertTrue(req.hasHeader("ACCEpt"));
assertEquals("Something, Everything", req.header("accept"));
assertTrue(req.hasHeader("fOO"));
assertEquals("Bar", req.header("foo"));
List accept = req.headers("accept");
assertEquals(2, accept.size());
assertEquals("Something", accept.get(0));
assertEquals("Everything", accept.get(1));
Map> headers = req.multiHeaders();
assertEquals(accept, headers.get("Accept"));
assertEquals("Bar", headers.get("Foo").get(0));
assertTrue(req.hasHeader("Accept"));
assertTrue(req.hasHeaderWithValue("accept", "Something"));
assertTrue(req.hasHeaderWithValue("accept", "Everything"));
assertFalse(req.hasHeaderWithValue("accept", "Something for nothing"));
req.removeHeader("accept");
headers = req.multiHeaders();
assertEquals("Bar", headers.get("Foo").get(0));
assertFalse(req.hasHeader("Accept"));
assertNull(headers.get("Accept"));
}
@Test void responseHeadersPreserveInsertOrder() throws IOException {
// linkedhashmap preserves the response header order
Connection.Response res = new HttpConnection.Response();
res.addHeader("5", "5");
res.addHeader("4", "4");
res.addHeader("3", "3");
res.addHeader("2", "2");
res.addHeader("1", "1");
String[] expected = {"5", "4", "3", "2", "1"};
int i = 0;
for (String key : res.headers().keySet()) {
assertEquals(expected[i++], key);
}
assertInstanceOf(LinkedHashMap.class, res.headers());
}
@Test public void ignoresEmptySetCookies() {
// prep http response header map
Map> headers = new HashMap<>();
headers.put("Set-Cookie", Collections.emptyList());
HttpConnection.Response res = new HttpConnection.Response();
res.processResponseHeaders(headers);
assertEquals(0, res.cookies().size());
}
@Test public void connectWithUrl() throws MalformedURLException {
Connection con = HttpConnection.connect(new URL("http://example.com"));
assertEquals("http://example.com", con.request().url().toExternalForm());
}
@Test public void throwsOnMalformedUrl() {
assertThrows(IllegalArgumentException.class, () -> HttpConnection.connect("bzzt"));
}
@Test public void userAgent() {
Connection con = HttpConnection.connect("http://example.com/");
assertEquals(HttpConnection.DEFAULT_UA, con.request().header("User-Agent"));
con.userAgent("Mozilla");
assertEquals("Mozilla", con.request().header("User-Agent"));
}
@Test public void timeout() {
Connection con = HttpConnection.connect("http://example.com/");
assertEquals(30 * 1000, con.request().timeout());
con.timeout(1000);
assertEquals(1000, con.request().timeout());
}
@Test public void referrer() {
Connection con = HttpConnection.connect("http://example.com/");
con.referrer("http://foo.com");
assertEquals("http://foo.com", con.request().header("Referer"));
}
@Test public void method() {
Connection con = HttpConnection.connect("http://example.com/");
assertEquals(Connection.Method.GET, con.request().method());
con.method(Connection.Method.POST);
assertEquals(Connection.Method.POST, con.request().method());
}
@Test public void throwsOnOddData() {
assertThrows(IllegalArgumentException.class, () -> {
Connection con = HttpConnection.connect("http://example.com/");
con.data("Name", "val", "what");
});
}
@Test public void data() {
Connection con = HttpConnection.connect("http://example.com/");
con.data("Name", "Val", "Foo", "bar");
Collection values = con.request().data();
Object[] data = values.toArray();
Connection.KeyVal one = (Connection.KeyVal) data[0];
Connection.KeyVal two = (Connection.KeyVal) data[1];
assertEquals("Name", one.key());
assertEquals("Val", one.value());
assertEquals("Foo", two.key());
assertEquals("bar", two.value());
}
@Test public void cookie() {
Connection con = HttpConnection.connect("http://example.com/");
con.cookie("Name", "Val");
assertEquals("Val", con.request().cookie("Name"));
}
@Test public void inputStream() {
Connection.KeyVal kv = HttpConnection.KeyVal.create("file", "thumb.jpg", ParseTest.inputStreamFrom("Check"));
assertEquals("file", kv.key());
assertEquals("thumb.jpg", kv.value());
assertTrue(kv.hasInputStream());
kv = HttpConnection.KeyVal.create("one", "two");
assertEquals("one", kv.key());
assertEquals("two", kv.value());
assertFalse(kv.hasInputStream());
}
@Test public void requestBody() {
Connection con = HttpConnection.connect("http://example.com/");
con.requestBody("foo");
assertEquals("foo", con.request().requestBody());
}
@Test public void encodeUrl() throws MalformedURLException {
URL url1 = new URL("https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag");
URL url2 = new UrlBuilder(url1).build();
assertEquals("https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag", url2.toExternalForm());
}
@Test public void encodeUrlSupplementary() throws MalformedURLException {
URL url1 = new URL("https://example.com/tools/test💩.html"); // = "/tools/test\uD83D\uDCA9.html"
URL url2 = new UrlBuilder(url1).build();
assertEquals("https://example.com/tools/test%F0%9F%92%A9.html", url2.toExternalForm());
}
@Test void encodedUrlDoesntDoubleEncode() throws MalformedURLException {
URL url1 = new URL("https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment");
URL url2 = new UrlBuilder(url1).build();
URL url3 = new UrlBuilder(url2).build();
assertEquals("https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment", url2.toExternalForm());
assertEquals("https://test.com/foo%20bar/%5BOne%5D?q=white+space#frag%20ment", url3.toExternalForm());
}
@Test void urlPathIsPreservedDoesntDoubleEncode() throws MalformedURLException {
URL url1 = new URL("https://test.com/[foo] bar+/%5BOne%5D?q=white space#frag ment");
URL url2 = new UrlBuilder(url1).build();
URL url3 = new UrlBuilder(url2).build();
assertEquals("https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment", url2.toExternalForm());
assertEquals("https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment", url3.toExternalForm());
}
@Test void connectToEncodedUrl() {
Connection connect = Jsoup.connect("https://example.com/a%20b%20c?query+string");
URL url = connect.request().url();
assertEquals("https://example.com/a%20b%20c?query+string", url.toExternalForm());
}
@Test void encodedUrlPathIsPreserved() {
// https://github.com/jhy/jsoup/issues/1952
Connection connect = Jsoup.connect("https://example.com/%2B32");
URL url = connect.request().url();
assertEquals("https://example.com/%2B32", url.toExternalForm());
}
@Test void urlPathPlusIsPreserved() {
// https://github.com/jhy/jsoup/issues/1952
Connection connect = Jsoup.connect("https://example.com/123+456");
URL url = connect.request().url();
assertEquals("https://example.com/123+456", url.toExternalForm());
}
@Test public void noUrlThrowsValidationError() throws IOException {
HttpConnection con = new HttpConnection();
boolean threw = false;
try {
con.execute();
} catch (IllegalArgumentException e) {
threw = true;
assertEquals("URL not set. Make sure to call #url(...) before executing the request.", e.getMessage());
}
assertTrue(threw);
}
@Test public void handlesHeaderEncodingOnRequest() {
Connection.Request req = new HttpConnection.Request();
req.addHeader("xxx", "é");
}
@Test public void supportsInternationalDomainNames() throws MalformedURLException {
String idn = "https://www.测试.测试/foo.html?bar";
String puny = "https://www.xn--0zwm56d.xn--0zwm56d/foo.html?bar";
Connection con = Jsoup.connect(idn);
assertEquals(puny, con.request().url().toExternalForm());
HttpConnection.Request req = new HttpConnection.Request();
req.url(new URL(idn));
assertEquals(puny, req.url().toExternalForm());
}
@Test void supportsIdnWithPort() throws MalformedURLException {
String idn = "https://www.测试.测试:9001/foo.html?bar";
String puny = "https://www.xn--0zwm56d.xn--0zwm56d:9001/foo.html?bar";
Connection con = Jsoup.connect(idn);
assertEquals(puny, con.request().url().toExternalForm());
HttpConnection.Request req = new HttpConnection.Request();
req.url(new URL(idn));
assertEquals(puny, req.url().toExternalForm());
}
@Test public void validationErrorsOnExecute() throws IOException {
Connection con = new HttpConnection();
boolean urlThrew = false;
try {
con.execute();
} catch (IllegalArgumentException e) {
urlThrew = e.getMessage().contains("URL");
}
assertTrue(urlThrew);
}
@Test void testMalformedException() {
boolean threw = false;
try {
Jsoup.connect("jsoup.org/test");
} catch (IllegalArgumentException e) {
threw = true;
assertEquals("The supplied URL, 'jsoup.org/test', is malformed. Make sure it is an absolute URL, and starts with 'http://' or 'https://'. See https://jsoup.org/cookbook/extracting-data/working-with-urls", e.getMessage());
}
assertTrue(threw);
}
@Test void setHeaderWithUnicodeValue() {
Connection connect = Jsoup.connect("https://example.com");
String value = "/foo/我的";
connect.header("Key", value);
String actual = connect.request().header("Key");
assertEquals(value, actual);
}
@Test void setAuth() throws MalformedURLException {
Connection con = Jsoup.newSession();
assertNull(con.request().auth());
RequestAuthenticator auth1 = new RequestAuthenticator() {
@Override public PasswordAuthentication authenticate(Context auth) {
return auth.credentials("foo", "bar");
}
};
RequestAuthenticator auth2 = new RequestAuthenticator() {
@Override public PasswordAuthentication authenticate(Context auth) {
return auth.credentials("qux", "baz");
}
};
con.auth(auth1);
assertSame(con.request().auth(), auth1);
con.auth(auth2);
assertSame(con.request().auth(), auth2);
con.request().auth(auth1);
assertSame(con.request().auth(), auth1);
PasswordAuthentication creds = auth1.authenticate(
new RequestAuthenticator.Context(new URL("http://example.com"), Authenticator.RequestorType.SERVER, "Realm"));
assertNotNull(creds);
assertEquals("foo", creds.getUserName());
assertEquals("bar", new String(creds.getPassword()));
}
/* Tests for fixHeaderEncoding. We are handling two cases when a server sends a header in UTF8. The JVM will decode
that in 8859 (per the RFC) and we'll get mojibake, and so we try to fix it. On Android, will be decoded correctly,
so should not be modified. */
static String mojibake(String input) {
// simulate mojibake by encoding in UTF-8 and decoding in ISO-8859-1
return new String(input.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1);
}
@ParameterizedTest
@ValueSource(strings = {"search.php?moji=我的", "latin=café", "🍕", "ascii"})
void fixesHeaderEncodingIfRequired(String input) {
// if the input was mojibaked, we fix it; otherwise is passed
// https://github.com/jhy/jsoup/issues/2011
assertEquals(input, fixHeaderEncoding(input));
assertEquals(input, fixHeaderEncoding(mojibake(input)));
}
}
================================================
FILE: src/test/java/org/jsoup/helper/RegexTest.java
================================================
package org.jsoup.helper;
import org.jsoup.select.QueryParser;
import org.jsoup.select.Selector;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.junit.jupiter.api.Assertions.*;
public class RegexTest {
private boolean originalUseRe2j; // track original setting
@BeforeEach
void setUp() {
originalUseRe2j = Regex.wantsRe2j();
}
@AfterEach
void tearDown() {
Regex.wantsRe2j(originalUseRe2j); // restore original setting
}
@ParameterizedTest
@ValueSource(booleans = {false, true})
void testRegexDelegates(boolean useRe2j) {
Regex.wantsRe2j(useRe2j);
assertEquals(Regex.usingRe2j(), useRe2j);
String pattern = "(\\d+)";
String input = "12345";
Regex regex = Regex.compile(pattern);
Regex.Matcher matcher = regex.matcher(input);
assertTrue(matcher.find());
}
@Test
void jdkSupportsBackreferenceMatches() {
Regex.wantsRe2j(false);
String pattern = "(\\w+)\\s+\\1"; // backreference to group 1
String input = "hello hello";
Regex regex = Regex.compile(pattern);
Regex.Matcher matcher = regex.matcher(input);
assertTrue(matcher.find());
}
@Test
void re2jRejectsBackreferenceThrows() {
Regex.wantsRe2j(true);
String pattern = "(\\w+)\\s+\\1"; // backreference unsupported by RE2J
assertThrows(ValidationException.class, () -> Regex.compile(pattern));
// and not the rej2 PatternSyntaxException
}
@ParameterizedTest
@ValueSource(booleans = {false, true})
void queryParserThrowsSelectorExceptionOnMalformedRegex(boolean useRe2j) {
Regex.wantsRe2j(useRe2j);
String query = "[attr~=(unclosed]";
boolean threw = false;
try {
QueryParser.parse(query);
} catch (Selector.SelectorParseException e) {
threw = true;
assertTrue(e.getMessage().contains("Pattern syntax error"));
}
assertTrue(threw);
}
}
================================================
FILE: src/test/java/org/jsoup/helper/ValidateTest.java
================================================
package org.jsoup.helper;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
@SuppressWarnings("deprecation") // keeps tests for ensureNotNull
public class ValidateTest {
@Test
public void testNotNull() {
Validate.notNull("foo");
boolean threw = false;
try {
Validate.notNull(null);
} catch (IllegalArgumentException e) {
threw = true;
}
Assertions.assertTrue(threw);
}
@Test
void stacktraceFiltersOutValidateClass() {
boolean threw = false;
try {
Validate.notNull(null);
} catch (ValidationException e) {
threw = true;
assertEquals("Object must not be null", e.getMessage());
StackTraceElement[] stackTrace = e.getStackTrace();
for (StackTraceElement trace : stackTrace) {
assertNotEquals(trace.getClassName(), Validate.class.getName());
}
assertTrue(stackTrace.length >= 1);
}
Assertions.assertTrue(threw);
}
@Test
void nonnullParam() {
boolean threw = true;
try {
Validate.notNullParam(null, "foo");
} catch (ValidationException e) {
assertEquals("The parameter 'foo' must not be null.", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testWtf() {
boolean threw = false;
try {
Validate.wtf("Unexpected state reached");
} catch (IllegalStateException e) {
threw = true;
assertEquals("Unexpected state reached", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testEnsureNotNull() {
// Test with a non-null object
Object obj = new Object();
assertSame(obj, Validate.ensureNotNull(obj));
// Test with a null object
boolean threw = false;
try {
Validate.ensureNotNull(null);
} catch (ValidationException e) {
threw = true;
assertEquals("Object must not be null", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testEnsureNotNullWithMessage() {
// Test with a non-null object
Object obj = new Object();
assertSame(obj, Validate.ensureNotNull(obj, "Object must not be null"));
// Test with a null object
boolean threw = false;
try {
Validate.ensureNotNull(null, "Custom error message");
} catch (ValidationException e) {
threw = true;
assertEquals("Custom error message", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testEnsureNotNullWithFormattedMessage() {
// Test with a non-null object
Object obj = new Object();
assertSame(obj, Validate.ensureNotNull(obj, "Object must not be null: %s", "additional info"));
// Test with a null object
boolean threw = false;
try {
Validate.ensureNotNull(null, "Object must not be null: %s", "additional info");
} catch (ValidationException e) {
threw = true;
assertEquals("Object must not be null: additional info", e.getMessage());
}
assertTrue(threw);
}
@Test void expectNotNull() {
String foo = "Foo";
String foo2 = Validate.expectNotNull(foo);
assertSame(foo, foo2);
// Test with a null object
String bar = null;
boolean threw = false;
try {
Validate.expectNotNull(bar);
} catch (ValidationException e) {
threw = true;
assertEquals("Object must not be null", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testNotNullParam() {
// Test with a non-null object
Object obj = new Object();
Validate.notNullParam(obj, "param");
// Test with a null object
boolean threw = false;
try {
Validate.notNullParam(null, "param");
} catch (ValidationException e) {
threw = true;
assertEquals("The parameter 'param' must not be null.", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testNotEmpty() {
// Test with a non-empty string
String str = "foo";
Validate.notEmpty(str);
// Test with an empty string
boolean threw = false;
try {
Validate.notEmpty("");
} catch (ValidationException e) {
threw = true;
assertEquals("String must not be empty", e.getMessage());
}
assertTrue(threw);
// Test with a null string
threw = false;
try {
Validate.notEmpty(null);
} catch (ValidationException e) {
threw = true;
assertEquals("String must not be empty", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testIsTrue() {
// Test with a true value
Validate.isTrue(true);
// Test with a false value
boolean threw = false;
try {
Validate.isTrue(false);
} catch (ValidationException e) {
threw = true;
assertEquals("Must be true", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testIsFalse() {
// Test with a false value
Validate.isFalse(false);
// Test with a true value
boolean threw = false;
try {
Validate.isFalse(true);
} catch (ValidationException e) {
threw = true;
assertEquals("Must be false", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testAssertFail() {
boolean result = false;
boolean threw = false;
try {
result = Validate.assertFail("This should fail");
} catch (ValidationException e) {
threw = true;
assertEquals("This should fail", e.getMessage());
}
assertTrue(threw);
assertFalse(result);
}
@Test
public void testNotEmptyParam() {
// Test with a non-empty string
Validate.notEmptyParam("foo", "param");
// Test with an empty string
boolean threw = false;
try {
Validate.notEmptyParam("", "param");
} catch (ValidationException e) {
threw = true;
assertEquals("The 'param' parameter must not be empty.", e.getMessage());
}
assertTrue(threw);
// Test with a null string
threw = false;
try {
Validate.notEmptyParam(null, "param");
} catch (ValidationException e) {
threw = true;
assertEquals("The 'param' parameter must not be empty.", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testNoNullElementsWithMessage() {
// Test with an array with no null elements
Object[] array = {new Object(), new Object()};
Validate.noNullElements(array, "Custom error message");
// Test with an array containing a null element
boolean threw = false;
try {
Validate.noNullElements(new Object[]{new Object(), null}, "Custom error message");
} catch (ValidationException e) {
threw = true;
assertEquals("Custom error message", e.getMessage());
}
assertTrue(threw);
}
@Test
public void testNotEmptyWithMessage() {
// Test with a non-empty string
Validate.notEmpty("foo", "Custom error message");
// Test with an empty string
boolean threw = false;
try {
Validate.notEmpty("", "Custom error message");
} catch (ValidationException e) {
threw = true;
assertEquals("Custom error message", e.getMessage());
}
assertTrue(threw);
// Test with a null string
threw = false;
try {
Validate.notEmpty(null, "Custom error message");
} catch (ValidationException e) {
threw = true;
assertEquals("Custom error message", e.getMessage());
}
assertTrue(threw);
}
}
================================================
FILE: src/test/java/org/jsoup/helper/W3CDomTest.java
================================================
package org.jsoup.helper;
import org.jsoup.Jsoup;
import org.jsoup.TextUtil;
import org.jsoup.integration.ParseTest;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Stream;
import static org.jsoup.TextUtil.normalizeSpaces;
import static org.jsoup.nodes.Document.OutputSettings.Syntax.xml;
import static org.junit.jupiter.api.Assertions.*;
public class W3CDomTest {
private static Document parseXml(String xml, boolean nameSpaceAware) {
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(nameSpaceAware);
DocumentBuilder builder = factory.newDocumentBuilder();
builder.setEntityResolver((publicId, systemId) -> {
if (systemId.contains("about:legacy-compat")) { //
return new InputSource(new StringReader(""));
} else {
return null;
}
});
Document dom = builder.parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
dom.normalizeDocument();
return dom;
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
@Test
public void simpleConversion() {
String html = "W3cText
What";
assertEquals(expected, TextUtil.stripNewlines(out));
Document roundTrip = parseXml(out, true);
assertEquals("Text", roundTrip.getElementsByTagName("p").item(0).getTextContent());
// check we can set properties
Map properties = W3CDom.OutputXml();
properties.put(OutputKeys.INDENT, "yes");
String furtherOut = W3CDom.asString(wDoc, properties);
assertTrue(furtherOut.length() > out.length()); // wanted to assert formatting, but actual indentation is platform specific so breaks in CI
String furtherExpected =
"W3cText
What";
assertEquals(furtherExpected, TextUtil.stripNewlines(furtherOut)); // on windows, DOM will write newlines as \r\n
}
@Test
public void namespacePreservation() throws IOException {
File in = ParseTest.getFile("/htmltests/namespaces.xhtml");
org.jsoup.nodes.Document jsoupDoc;
jsoupDoc = Jsoup.parse(in, "UTF-8", "", Parser.xmlParser());
Document doc;
org.jsoup.helper.W3CDom jDom = new org.jsoup.helper.W3CDom();
doc = jDom.fromJsoup(jsoupDoc);
Node htmlEl = doc.getChildNodes().item(0);
assertEquals("http://www.w3.org/1999/xhtml", htmlEl.getNamespaceURI());
assertEquals("html", htmlEl.getLocalName());
assertEquals("html", htmlEl.getNodeName());
// inherits default namespace
Node head = htmlEl.getFirstChild().getNextSibling();
assertEquals("http://www.w3.org/1999/xhtml", head.getNamespaceURI());
assertEquals("head", head.getLocalName());
assertEquals("head", head.getNodeName());
Node epubTitle = htmlEl.getChildNodes().item(3).getChildNodes().item(3);
assertEquals("Check", epubTitle.getTextContent());
assertEquals("http://www.idpf.org/2007/ops", epubTitle.getNamespaceURI());
assertEquals("title", epubTitle.getLocalName());
assertEquals("epub:title", epubTitle.getNodeName());
Node xSection = epubTitle.getNextSibling().getNextSibling();
assertEquals("urn:test", xSection.getNamespaceURI());
assertEquals("section", xSection.getLocalName());
assertEquals("x:section", xSection.getNodeName());
// https://github.com/jhy/jsoup/issues/977
// does not keep last set namespace
Node svg = xSection.getNextSibling().getNextSibling();
assertEquals("http://www.w3.org/2000/svg", svg.getNamespaceURI());
assertEquals("svg", svg.getLocalName());
assertEquals("svg", svg.getNodeName());
Node path = svg.getChildNodes().item(1);
assertEquals("http://www.w3.org/2000/svg", path.getNamespaceURI());
assertEquals("path", path.getLocalName());
assertEquals("path", path.getNodeName());
Node clip = path.getChildNodes().item(1);
assertEquals("http://example.com/clip", clip.getNamespaceURI());
assertEquals("clip", clip.getLocalName());
assertEquals("clip", clip.getNodeName());
assertEquals("456", clip.getTextContent());
Node picture = svg.getNextSibling().getNextSibling();
assertEquals("http://www.w3.org/1999/xhtml", picture.getNamespaceURI());
assertEquals("picture", picture.getLocalName());
assertEquals("picture", picture.getNodeName());
Node img = picture.getFirstChild();
assertEquals("http://www.w3.org/1999/xhtml", img.getNamespaceURI());
assertEquals("img", img.getLocalName());
assertEquals("img", img.getNodeName());
}
@Test
public void handlesInvalidAttributeNames() {
String html = "";
org.jsoup.nodes.Document jsoupDoc;
jsoupDoc = Jsoup.parse(html);
Element body = jsoupDoc.select("body").first();
assertTrue(body.hasAttr("\"")); // actually an attribute with key '"'. Correct per HTML5 spec, but w3c xml dom doesn't dig it
assertTrue(body.hasAttr("name\""));
Document w3Doc = W3CDom.convert(jsoupDoc);
String xml = W3CDom.asString(w3Doc, W3CDom.OutputXml());
assertEquals("", xml);
}
@Test
public void htmlInputDocMaintainsHtmlAttributeNames() {
String html = "unicode attr names
";
org.jsoup.nodes.Document jsoupDoc;
jsoupDoc = Jsoup.parse(html);
Document w3Doc = W3CDom.convert(jsoupDoc);
String out = W3CDom.asString(w3Doc, W3CDom.OutputHtml());
String expected = "unicode attr names
";
assertEquals(expected, TextUtil.stripNewlines(out));
}
@Test
public void xmlInputDocMaintainsHtmlAttributeNames() {
String html = "unicode attr names coerced
";
org.jsoup.nodes.Document jsoupDoc;
jsoupDoc = Jsoup.parse(html);
jsoupDoc.outputSettings().syntax(xml);
Document w3Doc = W3CDom.convert(jsoupDoc);
String out = W3CDom.asString(w3Doc, W3CDom.OutputHtml());
String expected = "unicode attr names coerced
";
assertEquals(expected, TextUtil.stripNewlines(out));
}
@Test
public void handlesInvalidTagAsText() {
org.jsoup.nodes.Document jsoup = Jsoup.parse("<インセンティブで高収入！>Text More
");
Document w3Doc = W3CDom.convert(jsoup);
String xml = W3CDom.asString(w3Doc, W3CDom.OutputXml());
assertEquals("<インセンティブで高収入！>Text More
", xml);
}
@Test void handlesHtmlElsWithLt() {
// In HTML, elements can be named "foo). Test that we can convert to W3C, that we can HTML parse our HTML serial, XML parse our XML serial, and W3C XML parse the XML serial and the W3C serial
// And similarly attributes may have "<" in their name
// https://github.com/jhy/jsoup/issues/2259
String input = "Text";
String xmlExpect = "Text"; // rewrites < to _ in el and attr
// html round trips
org.jsoup.nodes.Document htmlDoc = Jsoup.parse(input);
String htmlSerial = htmlDoc.body().html();
assertEquals(input, normalizeSpaces(htmlSerial)); // same as input
Element htmlRound = Jsoup.parse(htmlSerial).body();
assertTrue(htmlDoc.body().hasSameValue(htmlRound));
// xml round trips
htmlDoc.outputSettings().syntax(xml);
String asXml = htmlDoc.body().html();
assertEquals(xmlExpect, normalizeSpaces(asXml)); // ->
org.jsoup.nodes.Document xmlDoc = Jsoup.parse(asXml);
String xmlSerial = xmlDoc.body().html();
assertEquals(xmlExpect, normalizeSpaces(xmlSerial)); // same as xmlExpect
Element xmlRound = Jsoup.parse(xmlSerial).body();
assertTrue(xmlDoc.body().hasSameValue(xmlRound));
// Can W3C parse that XML
Document w3cXml = parseXml(asXml, true);
NodeList w3cXmlNodes = w3cXml.getElementsByTagName("foo_bar");
assertEquals(1, w3cXmlNodes.getLength());
assertEquals("123", w3cXmlNodes.item(0).getAttributes().getNamedItem("attr_name").getTextContent());
// Can convert to W3C
Document w3cDoc = W3CDom.convert(htmlDoc);
NodeList w3cNodes = w3cDoc.getElementsByTagName("foo_bar");
assertEquals(1, w3cNodes.getLength());
assertEquals("123", w3cNodes.item(0).getAttributes().getNamedItem("attr_name").getTextContent());
}
@Test
public void canConvertToCustomDocument() throws ParserConfigurationException {
org.jsoup.nodes.Document document = Jsoup.parse("");
DocumentBuilderFactory localDocumentBuilderFactory = DocumentBuilderFactory.newInstance();
Document customDocumentResult = localDocumentBuilderFactory.newDocumentBuilder().newDocument();
W3CDom w3cDom = new W3CDom();
w3cDom.convert(document, customDocumentResult);
String html = W3CDom.asString(customDocumentResult, W3CDom.OutputHtml());
assertEquals("", html);
}
@Test
public void treatsUndeclaredNamespaceAsLocalName() {
String html = "One";
org.jsoup.nodes.Document doc = Jsoup.parse(html);
Document w3Doc = new W3CDom().fromJsoup(doc);
Node htmlEl = w3Doc.getFirstChild();
assertEquals("http://www.w3.org/1999/xhtml", htmlEl.getNamespaceURI());
assertEquals("html", htmlEl.getLocalName());
assertEquals("html", htmlEl.getNodeName());
Node fb = htmlEl.getFirstChild().getNextSibling().getFirstChild();
assertEquals("http://www.w3.org/1999/xhtml", fb.getNamespaceURI());
assertEquals("like", fb.getLocalName());
assertEquals("fb:like", fb.getNodeName());
}
@Test
public void xmlnsXpathTest() throws XPathExpressionException {
W3CDom w3c = new W3CDom();
String html = "hello
";
Document dom = w3c.fromJsoup(Jsoup.parse(html));
NodeList nodeList = xpath(dom, "//*[local-name()=\"body\"]");// namespace aware; HTML namespace is default
assertEquals("div", nodeList.item(0).getLocalName());
// default output is namespace aware, so query needs to be as well
html = "hello
";
dom = w3c.fromJsoup(Jsoup.parse(html));
nodeList = xpath(dom, "//body");
assertNull(nodeList); // no matches
dom = w3c.fromJsoup(Jsoup.parse(html));
nodeList = xpath(dom, "//*[local-name()=\"body\"]");
assertNotNull(nodeList);
assertEquals(1, nodeList.getLength());
assertEquals("div", nodeList.item(0).getLocalName());
assertEquals("http://www.w3.org/1999/xhtml", nodeList.item(0).getNamespaceURI());
assertNull(nodeList.item(0).getPrefix());
// get rid of the name space awareness
String xml = w3c.asString(dom);
dom = parseXml(xml, false);
Node item = (Node) xpath(dom, "//body");
assertEquals("body", item.getNodeName());
assertNull(item.getNamespaceURI());
assertNull(item.getPrefix());
// put back, will get zero
dom = parseXml(xml, true);
nodeList = xpath(dom, "//body");
assertNull(nodeList);
}
@Test
public void xhtmlNoNamespace() throws XPathExpressionException {
W3CDom w3c = new W3CDom();
String html = "hello
";
w3c.namespaceAware(false);
Document dom = w3c.fromJsoup(Jsoup.parse(html));
NodeList nodeList = xpath(dom, "//body");// no namespace
assertEquals(1, nodeList.getLength());
assertEquals("div", nodeList.item(0).getLocalName());
}
@Test
void canDisableNamespaces() throws XPathExpressionException {
W3CDom w3c = new W3CDom();
assertTrue(w3c.namespaceAware());
w3c.namespaceAware(false);
assertFalse(w3c.namespaceAware());
String html = "hello
";
Document dom = w3c.fromJsoup(Jsoup.parse(html));
NodeList nodeList = xpath(dom, "//body");// no ns, so needs no prefix
assertEquals("div", nodeList.item(0).getLocalName());
}
private NodeList xpath(Document w3cDoc, String query) throws XPathExpressionException {
XPathExpression xpath = XPathFactory.newInstance().newXPath().compile(query);
return ((NodeList) xpath.evaluate(w3cDoc, XPathConstants.NODE));
}
@Test
public void testRoundTripDoctype() {
// because we have Saxon on the test classpath, the transformer will change to that, and so case may change (e.g. Java base is META, Saxon is meta for HTML)
String base = "One
";
assertEqualsIgnoreCase("One
", output(base, true));
assertEqualsIgnoreCase("One
", output(base, false));
String publicDoc = "";
assertEqualsIgnoreCase("", output(publicDoc, true));
// different impls will have different XML formatting. OpenJDK 13 default gives this: but others have , so just check start
assertTrue(output(publicDoc, false).startsWith("";
assertEqualsIgnoreCase("", output(systemDoc, true));
assertEqualsIgnoreCase("", output(systemDoc, false));
String legacyDoc = "";
assertEqualsIgnoreCase("", output(legacyDoc, true));
assertEqualsIgnoreCase("", output(legacyDoc, false));
String noDoctype = "One
";
assertEqualsIgnoreCase("One
", output(noDoctype, true));
assertEqualsIgnoreCase("One
", output(noDoctype, false));
}
private String output(String in, boolean modeHtml) {
org.jsoup.nodes.Document jdoc = Jsoup.parse(in);
Document w3c = W3CDom.convert(jdoc);
Map properties = modeHtml ? W3CDom.OutputHtml() : W3CDom.OutputXml();
return normalizeSpaces(W3CDom.asString(w3c, properties));
}
private void assertEqualsIgnoreCase(String want, String have) {
assertEquals(want.toLowerCase(Locale.ROOT), have.toLowerCase(Locale.ROOT));
}
@Test
public void canOutputHtmlWithoutNamespace() {
String html = "One
";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
W3CDom w3c = new W3CDom();
w3c.namespaceAware(false);
String asHtml = W3CDom.asString(w3c.fromJsoup(jdoc), W3CDom.OutputHtml());
String asXtml = W3CDom.asString(w3c.fromJsoup(jdoc), W3CDom.OutputXml());
assertEqualsIgnoreCase(
"one
",
asHtml);
assertEqualsIgnoreCase(
"One
",
asXtml);
}
@Test public void convertsElementsAndMaintainsSource() {
org.jsoup.nodes.Document jdoc = Jsoup.parse("Two");
W3CDom w3CDom = new W3CDom();
Element jDiv = jdoc.selectFirst("div");
assertNotNull(jDiv);
Document doc = w3CDom.fromJsoup(jDiv);
Node div = w3CDom.contextNode(doc);
assertEquals("div", div.getLocalName());
assertEquals(jDiv, div.getUserData(W3CDom.SourceProperty));
Node textNode = div.getFirstChild().getFirstChild();
assertEquals("One", textNode.getTextContent());
assertEquals(Node.TEXT_NODE, textNode.getNodeType());
org.jsoup.nodes.TextNode jText = (TextNode) jDiv.childNode(0).childNode(0);
assertEquals(jText, textNode.getUserData(W3CDom.SourceProperty));
}
@Test public void canXmlParseCdataNodes() throws XPathExpressionException {
String html = "
5 && 6
";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
jdoc.outputSettings().syntax(xml);
String xml = jdoc.body().html();
assertTrue(xml.contains("")); // as asserted in ElementTest
Document doc = parseXml(xml, false);
NodeList list = xpath(doc, "//script");
assertEquals(2, list.getLength());
Node scriptComment = list.item(0); // will be the cdata node
assertEquals("//", scriptComment.getTextContent());
Node script = list.item(1);
assertEquals("\n" +
"1 && 2\n" +
"//", script.getTextContent());
}
@Test public void handlesEmptyDoctype() {
String html = "Foo";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
Document doc = (new W3CDom()).fromJsoup(jdoc);
assertNull(doc.getDoctype());
assertEquals("Foo", doc.getFirstChild().getTextContent());
}
@Test void testHtmlParseAttributesAreCaseInsensitive() throws IOException {
// https://github.com/jhy/jsoup/issues/981
String html = "\n" +
"\n" +
"
![\"Alt](\"firstImage.jpg\")
\n" +
"
![\"Alt](\"secondImage.jpg\")
\n" +
"\n" +
"";
org.jsoup.nodes.Document jsoupDoc;
jsoupDoc = Jsoup.parse(html);
org.jsoup.helper.W3CDom jDom = new org.jsoup.helper.W3CDom();
Document doc = jDom.fromJsoup(jsoupDoc);
org.w3c.dom.Element body = (org.w3c.dom.Element) doc.getDocumentElement().getElementsByTagName("body").item(0);
NodeList imgs = body.getElementsByTagName("img");
assertEquals(2, imgs.getLength());
org.w3c.dom.Element first = (org.w3c.dom.Element) imgs.item(0);
assertEquals(first.getAttributes().getLength(), 2);
String img1 = first.getAttribute("src");
assertEquals("firstImage.jpg", img1);
String alt1 = first.getAttribute("alt");
assertEquals("Alt one", alt1);
org.w3c.dom.Element second = (org.w3c.dom.Element) imgs.item(1);
assertEquals(second.getAttributes().getLength(), 2);
String img2 = second.getAttribute("src");
assertEquals("secondImage.jpg", img2);
String alt2 = second.getAttribute("alt");
assertEquals("Alt two", alt2);
}
@ParameterizedTest
@MethodSource("parserProvider")
void doesNotExpandEntities(Parser parser) {
// Tests that the billion laughs attack doesn't expand entities; also for XXE
// Not impacted because jsoup doesn't parse the entities within the doctype, and so won't get to the w3c.
// Added to confirm, and catch if that ever changes
String billionLaughs = "\n" +
"\n" +
" \n" +
"]>\n" +
"
&lol1;
";
org.jsoup.nodes.Document jsoupDoc = Jsoup.parse(billionLaughs, parser);
W3CDom w3cDom = new W3CDom();
org.w3c.dom.Document w3cDoc = w3cDom.fromJsoup(jsoupDoc);
assertNotNull(w3cDoc);
// select the p and make sure it's unexpanded
NodeList p = w3cDoc.getElementsByTagName("p");
assertEquals(1, p.getLength());
assertEquals("&lol1;", p.item(0).getTextContent());
// Check the string
String string = W3CDom.asString(w3cDoc, W3CDom.OutputXml());
assertFalse(string.contains("lololol"));
assertTrue(string.contains("&lol1;"));
}
@Test void undeclaredAttrNamespaceAsString() {
// https://github.com/jhy/jsoup/issues/2087
W3CDom w3CDom = new W3CDom();
String html = "
";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);
String xml = w3CDom.asString(w3CDoc);
assertEquals("
", xml);
}
@Test void declaredNamespaceIsUsed() {
W3CDom w3CDom = new W3CDom();
String html = "
";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);
String xml = w3CDom.asString(w3CDoc);
assertEquals("
", xml);
}
@Test void nestedElementsWithUndeclaredNamespace() {
W3CDom w3CDom = new W3CDom();
String html = "
";
org.jsoup.nodes.Document jdoc = Jsoup.parse(html);
org.w3c.dom.Document w3CDoc = w3CDom.fromJsoup(jdoc);
String xml = w3CDom.asString(w3CDoc);
assertEquals("
", xml);
}
private static Stream parserProvider() {
return Stream.of(
Arguments.of(Parser.htmlParser()),
Arguments.of(Parser.xmlParser())
);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/Benchmark.java
================================================
package org.jsoup.integration;
import java.util.Date;
/**
Does an A/B test on two methods, and prints out how long each took.
@author Jonathan Hedley, jonathan@hedley.net */
public class Benchmark {
public static void run(Runnable a, Runnable b, int count) {
long aMillis;
long bMillis;
print("Running test A (x%d)", count);
aMillis = time(a, count);
print("Running test B");
bMillis = time(b, count);
print("\nResults:");
print("A: %.2fs", aMillis / 1000f);
print("B: %.2fs", bMillis / 1000f);
print("\nB ran in %.2f %% time of A\n", (bMillis *1f / aMillis * 1f) * 100f);
}
private static long time(Runnable test, int count) {
Date start = new Date();
for (int i = 0; i < count; i++) {
test.run();
}
Date end = new Date();
return end.getTime() - start.getTime();
}
private static void print(String msgFormat, Object... msgParams) {
System.out.println(String.format(msgFormat, msgParams));
}
}
================================================
FILE: src/test/java/org/jsoup/integration/ConnectIT.java
================================================
package org.jsoup.integration;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.helper.DataUtil;
import org.jsoup.integration.servlets.EchoServlet;
import org.jsoup.integration.servlets.FileServlet;
import org.jsoup.integration.servlets.SlowRider;
import org.jsoup.internal.SharedConstants;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.StreamParser;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.*;
/**
* Failsafe integration tests for Connect methods. These take a bit longer to run, so included as Integ, not Unit, tests.
*/
public class ConnectIT {
@BeforeAll
public static void setUp() {
TestServer.start();
System.setProperty(SharedConstants.UseHttpClient, "false"); // use the default UrlConnection. See HttpClientConnectIT for other version
}
// Slow Rider tests.
@Test
public void canInterruptBodyStringRead() throws InterruptedException {
final String[] body = new String[1];
Thread runner = new Thread(() -> {
try {
Connection.Response res = Jsoup.connect(SlowRider.Url)
.timeout(15 * 1000)
.execute();
body[0] = res.body();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
runner.start();
Thread.sleep(1000 * 3);
runner.interrupt();
assertTrue(runner.isInterrupted());
runner.join();
assertTrue(body[0].length() > 0);
assertTrue(body[0].contains("Are you still there?"));
}
@Test
public void canInterruptDocumentRead() throws InterruptedException {
long start = System.currentTimeMillis();
final String[] body = new String[1];
Thread runner = new Thread(() -> {
try {
Connection.Response res = Jsoup.connect(SlowRider.Url)
.timeout(15 * 1000)
.execute();
body[0] = res.parse().text();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
runner.start();
Thread.sleep(3 * 1000);
runner.interrupt();
assertTrue(runner.isInterrupted());
runner.join();
long end = System.currentTimeMillis();
// check we are between 3 and connect timeout seconds (should be just over 3; but allow some slack for slow CI runners)
assertTrue(end - start > 3 * 1000);
assertTrue(end - start < 10 * 1000);
}
@Test public void canInterruptThenJoinASpawnedThread() throws InterruptedException {
// https://github.com/jhy/jsoup/issues/1991
AtomicBoolean ioException = new AtomicBoolean();
Thread runner = new Thread(() -> {
try {
while (!Thread.currentThread().isInterrupted()) {
Document doc = Jsoup.connect(SlowRider.Url)
.timeout(30000)
.get();
}
} catch (IOException e) {
ioException.set(true); // don't expect to catch, because the outer sleep will complete before this timeout
}
});
runner.start();
Thread.sleep(2 * 1000);
runner.interrupt();
runner.join();
assertFalse(ioException.get());
}
@Test
public void totalTimeout() throws IOException {
int timeout = 3 * 1000;
long start = System.currentTimeMillis();
boolean threw = false;
try {
Jsoup.connect(SlowRider.Url).timeout(timeout).get();
} catch (SocketTimeoutException e) {
long end = System.currentTimeMillis();
long took = end - start;
assertTrue(took > timeout, ("Time taken was " + took));
assertTrue(took < timeout * 1.8, ("Time taken was " + took));
threw = true;
}
assertTrue(threw);
}
@Test
public void slowReadOk() throws IOException {
// make sure that a slow read that is under the request timeout is still OK
Document doc = Jsoup.connect(SlowRider.Url)
.data(SlowRider.MaxTimeParam, "2000") // the request completes in 2 seconds
.get();
Element h1 = doc.selectFirst("h1");
assertEquals("outatime", h1.text());
}
@Test void readFullyThrowsOnTimeout() throws IOException {
// tests that response.readFully excepts on timeout
boolean caught = false;
Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();
try {
res.readFully();
} catch (IOException e) {
caught = true;
}
assertTrue(caught);
}
@Test void readBodyThrowsOnTimeout() throws IOException {
// tests that response.readBody excepts on timeout
boolean caught = false;
Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();
try {
res.readBody();
} catch (IOException e) {
caught = true;
}
assertTrue(caught);
}
@Test void bodyThrowsUncheckedOnTimeout() throws IOException {
// tests that response.body unchecked excepts on timeout
boolean caught = false;
Connection.Response res = Jsoup.connect(SlowRider.Url).timeout(3000).execute();
try {
res.body();
} catch (UncheckedIOException e) {
caught = true;
}
assertTrue(caught);
}
@Test
public void infiniteReadSupported() throws IOException {
Document doc = Jsoup.connect(SlowRider.Url)
.timeout(0)
.data(SlowRider.MaxTimeParam, "2000")
.get();
Element h1 = doc.selectFirst("h1");
assertEquals("outatime", h1.text());
}
@Test void streamParserUncheckedExceptionOnTimeoutInStream() throws IOException {
boolean caught = false;
try (StreamParser streamParser = Jsoup.connect(SlowRider.Url)
.data(SlowRider.MaxTimeParam, "10000")
.data(SlowRider.IntroSizeParam, "8000") // 8K to pass first buffer, or the timeout would occur in execute or streamparser()
.timeout(4000) // has a 1000 sleep at the start
.execute()
.streamParser()) {
// we should expect to timeout while in stream
try {
long count = streamParser.stream().count();
} catch (Exception e) {
caught = true;
UncheckedIOException ioe = (UncheckedIOException) e;
IOException cause = ioe.getCause();
//assertInstanceOf(SocketTimeoutException.class, cause); // different JDKs seem to wrap this differently
assertInstanceOf(IOException.class, cause);
}
}
assertTrue(caught);
}
@Test void streamParserCheckedExceptionOnTimeoutInSelect() throws IOException {
boolean caught = false;
try (StreamParser streamParser = Jsoup.connect(SlowRider.Url)
.data(SlowRider.MaxTimeParam, "10000")
.data(SlowRider.IntroSizeParam, "8000") // 8K to pass first buffer, or the timeout would occur in execute or streamparser()
.timeout(4000) // has a 1000 sleep at the start
.execute()
.streamParser()) {
// we should expect to timeout while in stream
try {
long count = 0;
while (streamParser.selectNext("p") != null) {
count++;
}
} catch (IOException e) {
caught = true;
}
}
assertTrue(caught);
}
private static final int LargeHtmlSize = 280735;
@Test
public void remainingAfterFirstRead() throws IOException {
int bufferSize = 5 * 1024;
int capSize = 100 * 1024;
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
try (BufferedInputStream stream = Jsoup.connect(url).maxBodySize(capSize)
.execute().bodyStream()) {
// simulates parse which does a limited read first
stream.mark(bufferSize);
ByteBuffer firstBytes = DataUtil.readToByteBuffer(stream, bufferSize);
byte[] array = firstBytes.array();
String firstText = new String(array, StandardCharsets.UTF_8);
assertTrue(firstText.startsWith("
Large"));
assertEquals(bufferSize, array.length);
boolean fullyRead = stream.read() == -1;
assertFalse(fullyRead);
// reset and read again
stream.reset();
ByteBuffer fullRead = DataUtil.readToByteBuffer(stream, 0);
byte[] fullArray = fullRead.array();
// bodyStream is not capped to body size - only for jsoup consumed stream
assertTrue(fullArray.length > capSize);
assertEquals(LargeHtmlSize, fullRead.limit());
String fullText = new String(fullRead.array(), 0, fullRead.limit(), StandardCharsets.UTF_8);
assertTrue(fullText.startsWith(firstText));
assertEquals(LargeHtmlSize, fullText.length());
}
}
@Test
public void noLimitAfterFirstRead() throws IOException {
int firstMaxRead = 5 * 1024;
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
try (BufferedInputStream stream = Jsoup.connect(url).execute().bodyStream()) {
// simulates parse which does a limited read first
stream.mark(firstMaxRead);
ByteBuffer firstBytes = DataUtil.readToByteBuffer(stream, firstMaxRead);
byte[] array = firstBytes.array();
String firstText = new String(array, StandardCharsets.UTF_8);
assertTrue(firstText.startsWith("<html><head><title>Large"));
assertEquals(firstMaxRead, array.length);
// reset and read fully
stream.reset();
ByteBuffer fullRead = DataUtil.readToByteBuffer(stream, 0);
assertEquals(LargeHtmlSize, fullRead.limit());
String fullText = new String(fullRead.array(), 0, fullRead.limit(), StandardCharsets.UTF_8);
assertTrue(fullText.startsWith(firstText));
assertEquals(LargeHtmlSize, fullText.length());
}
}
@Test public void bodyStreamConstrainedViaReadFully() throws IOException {
int cap = 5 * 1024;
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
try (BufferedInputStream stream = Jsoup
.connect(url)
.maxBodySize(cap)
.execute()
.readFully()
.bodyStream()) {
ByteBuffer cappedRead = DataUtil.readToByteBuffer(stream, 0);
assertEquals(cap, cappedRead.limit());
}
}
@Test public void bodyStreamConstrainedViaBufferUp() throws IOException {
int cap = 5 * 1024;
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
try (BufferedInputStream stream = Jsoup
.connect(url)
.maxBodySize(cap)
.execute()
.bufferUp()
.bodyStream()) {
ByteBuffer cappedRead = DataUtil.readToByteBuffer(stream, 0);
assertEquals(cap, cappedRead.limit());
}
}
}
================================================
FILE: src/test/java/org/jsoup/integration/ConnectTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Connection;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Method;
import org.jsoup.TextUtil;
import org.jsoup.UnsupportedMimeTypeException;
import org.jsoup.helper.DataUtil;
import org.jsoup.helper.W3CDom;
import org.jsoup.integration.servlets.*;
import org.jsoup.internal.SharedConstants;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.FormElement;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.XmlDeclaration;
import org.jsoup.parser.HtmlTreeBuilder;
import org.jsoup.parser.Parser;
import org.jsoup.parser.StreamParser;
import org.jsoup.parser.XmlTreeBuilder;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import static org.jsoup.helper.AuthenticationHandlerTest.MaxAttempts;
import static org.jsoup.helper.HttpConnection.CONTENT_TYPE;
import static org.jsoup.helper.HttpConnection.MULTIPART_FORM_DATA;
import static org.junit.jupiter.api.Assertions.*;
/**
* Tests Jsoup.connect against a local server.
*/
public class ConnectTest {
private static final int LargeDocFileLen = 280735;
private static final int LargeDocTextLen = 269535;
private static String echoUrl;
@BeforeAll
public static void setUp() {
TestServer.start();
echoUrl = EchoServlet.Url;
System.setProperty(SharedConstants.UseHttpClient, "false"); // use the default UrlConnection. See HttpClientConnectTest for other version
}
@BeforeEach
public void emptyCookieJar() {
// empty the cookie jar, so cookie tests are independent.
Jsoup.connect("http://example.com").cookieStore().removeAll();
}
@Test
public void canConnectToLocalServer() throws IOException {
String url = HelloServlet.Url;
Document doc = Jsoup.connect(url).get();
Element p = doc.selectFirst("p");
assertEquals("Hello, World!", p.text());
}
@Test void canConnectToLocalTlsServer() throws IOException {
String url = HelloServlet.TlsUrl;
Document doc = Jsoup.connect(url).get();
Element p = doc.selectFirst("p");
assertEquals("Hello, World!", p.text());
}
@Test
public void fetchURl() throws IOException {
Document doc = Jsoup.parse(new URL(echoUrl), 10 * 1000);
assertTrue(doc.title().contains("Environment Variables"));
}
@Test
public void fetchURIWithWhitespace() throws IOException {
Connection con = Jsoup.connect(echoUrl + "#with whitespaces");
Document doc = con.get();
assertTrue(doc.title().contains("Environment Variables"));
}
@Test
public void exceptOnUnsupportedProtocol() {
String url = "file://etc/passwd";
boolean threw = false;
try {
Document doc = Jsoup.connect(url).get();
} catch (MalformedURLException e) {
threw = true;
assertEquals("java.net.MalformedURLException: Only http & https protocols supported", e.toString());
} catch (IOException e) {
}
assertTrue(threw);
}
static String ihVal(String key, Document doc) {
final Element first = doc.select("th:contains(" + key + ") + td").first();
return first != null ? first.text() : null;
}
@Test void statusMessage() throws IOException {
Connection con = Jsoup.connect(EchoServlet.Url);
Document doc = con.get();
assertEquals("OK", con.response().statusMessage());
}
@Test
public void throwsExceptionOn404() {
String url = EchoServlet.Url;
Connection con = Jsoup.connect(url).header(EchoServlet.CodeParam, "404");
boolean threw = false;
try {
Document doc = con.get();
} catch (HttpStatusException e) {
threw = true;
assertEquals("org.jsoup.HttpStatusException: HTTP error fetching URL. Status=404, URL=[" + e.getUrl() + "]", e.toString());
assertTrue(e.getUrl().startsWith(url));
assertEquals(404, e.getStatusCode());
} catch (IOException e) {
}
assertTrue(threw);
}
@Test
public void ignoresExceptionIfSoConfigured() throws IOException {
String url = EchoServlet.Url;
Connection con = Jsoup.connect(url)
.header(EchoServlet.CodeParam, "404")
.ignoreHttpErrors(true);
Connection.Response res = con.execute();
Document doc = res.parse();
assertEquals(404, res.statusCode());
assertEquals("Not Found", res.statusMessage());
assertEquals("Webserver Environment Variables", doc.title());
}
@Test
public void doesPost() throws IOException {
Document doc = Jsoup.connect(echoUrl)
.data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
.cookie("auth", "token")
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("gzip", ihVal("Accept-Encoding", doc));
assertEquals("auth=token", ihVal("Cookie", doc));
assertEquals("度一下", ihVal("百", doc));
assertEquals("Jsoup, Jonathan", ihVal("uname", doc));
assertEquals("application/x-www-form-urlencoded; charset=UTF-8", ihVal("Content-Type", doc));
}
@Test
public void doesPostMultipartWithoutInputstream() throws IOException {
Document doc = Jsoup.connect(echoUrl)
.header(CONTENT_TYPE, MULTIPART_FORM_DATA)
.data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
.post();
assertTrue(ihVal("Content-Type", doc).contains(MULTIPART_FORM_DATA));
assertTrue(ihVal("Content-Type", doc).contains("boundary")); // should be automatically set
assertEquals("Jsoup, Jonathan", ihVal("uname", doc));
assertEquals("度一下", ihVal("百", doc));
}
@Test
public void canSendSecFetchHeaders() throws IOException {
// https://github.com/jhy/jsoup/issues/1461
Document doc = Jsoup.connect(echoUrl)
.header("Random-Header-name", "hello")
.header("Sec-Fetch-Site", "cross-site")
.header("Sec-Fetch-Mode", "cors")
.get();
assertEquals("hello", ihVal("Random-Header-name", doc));
assertEquals("cross-site", ihVal("Sec-Fetch-Site", doc));
assertEquals("cors", ihVal("Sec-Fetch-Mode", doc));
}
@Test
public void secFetchHeadersSurviveRedirect() throws IOException {
Document doc = Jsoup
.connect(RedirectServlet.Url)
.data(RedirectServlet.LocationParam, echoUrl)
.header("Random-Header-name", "hello")
.header("Sec-Fetch-Site", "cross-site")
.header("Sec-Fetch-Mode", "cors")
.get();
assertEquals("hello", ihVal("Random-Header-name", doc));
assertEquals("cross-site", ihVal("Sec-Fetch-Site", doc));
assertEquals("cors", ihVal("Sec-Fetch-Mode", doc));
}
@Test
public void sendsRequestBodyJsonWithData() throws IOException {
final String body = "{key:value}";
Document doc = Jsoup.connect(echoUrl)
.requestBody(body)
.header("Content-Type", "application/json")
.data("foo", "true")
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("application/json", ihVal("Content-Type", doc));
assertEquals("foo=true", ihVal("Query String", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@Test
public void sendsRequestBodyJsonWithoutData() throws IOException {
final String body = "{key:value}";
Document doc = Jsoup.connect(echoUrl)
.requestBody(body)
.header("Content-Type", "application/json")
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("application/json", ihVal("Content-Type", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@Test
public void sendsRequestBody() throws IOException {
final String body = "{key:value}";
Document doc = Jsoup.connect(echoUrl)
.requestBody(body)
.header("Content-Type", "text/plain")
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("text/plain", ihVal("Content-Type", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@Test
public void sendsRequestBodyWithUrlParams() throws IOException {
final String body = "{key:value}";
Document doc = Jsoup.connect(echoUrl)
.requestBody(body)
.data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
.header("Content-Type", "text/plain") // todo - if user sets content-type, we should append postcharset
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("uname=Jsoup&uname=Jonathan&%E7%99%BE=%E5%BA%A6%E4%B8%80%E4%B8%8B", ihVal("Query String", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@Test void sendsRequestBodyStream() throws IOException {
final String body = "{key:value}";
InputStream stream = new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8));
Document doc = Jsoup.connect(echoUrl)
.requestBodyStream(stream)
.header("Content-Type", "application/json")
.data("foo", "true")
.post();
assertEquals("POST", ihVal("Method", doc));
assertEquals("application/json", ihVal("Content-Type", doc));
assertEquals("foo=true", ihVal("Query String", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@ParameterizedTest @MethodSource("echoUrls") // http and https
public void doesGet(String url) throws IOException {
Connection con = Jsoup.connect(url + "?what=the")
.userAgent("Mozilla")
.referrer("http://example.com")
.data("what", "about & me?");
Document doc = con.get();
assertEquals("what=the&what=about+%26+me%3F", ihVal("Query String", doc));
assertEquals("the, about & me?", ihVal("what", doc));
assertEquals("Mozilla", ihVal("User-Agent", doc));
assertEquals("http://example.com", ihVal("Referer", doc));
}
@ParameterizedTest @MethodSource("echoUrls") // http and https
public void streamParserGet(String url) throws IOException {
Connection con = Jsoup.connect(url)
.userAgent("Mozilla")
.referrer("http://example.com")
.data("what", "about & me?");
//final Element first = doc.select("th:contains(" + key + ") + td").first();
try (StreamParser streamer = con.execute().streamParser()) {
Element title = streamer.expectFirst("title");
assertEquals("Webserver Environment Variables", title.text());
Element method = streamer.expectNext(echoSelect("Method"));
assertEquals("GET", method.text());
Document doc = streamer.document();
assertSame(doc, title.ownerDocument());
assertEquals(url + "?what=about+%26+me%3F", doc.location()); // with the query string
}
}
static String echoSelect(String key) {
return String.format("th:contains(%s) + td", key);
}
@Test
public void doesPut() throws IOException {
Connection.Response res = Jsoup.connect(echoUrl)
.data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
.cookie("auth", "token")
.method(Connection.Method.PUT)
.execute();
Document doc = res.parse();
assertEquals("PUT", ihVal("Method", doc));
assertEquals("gzip", ihVal("Accept-Encoding", doc));
assertEquals("auth=token", ihVal("Cookie", doc));
}
@Test
public void doesDeleteWithBody() throws IOException {
// https://github.com/jhy/jsoup/issues/1972
String body = "some body";
Connection.Response res = Jsoup.connect(echoUrl)
.requestBody(body)
.method(Method.DELETE)
.execute();
Document doc = res.parse();
assertEquals("DELETE", ihVal("Method", doc));
assertEquals(body, ihVal("Post Data", doc));
}
@Test
public void doesDeleteWithoutBody() throws IOException {
Connection.Response res = Jsoup.connect(echoUrl)
.method(Method.DELETE)
.execute();
Document doc = res.parse();
assertEquals("DELETE", ihVal("Method", doc));
assertEquals(null, ihVal("Post Data", doc));
}
/**
* Tests upload of content to a remote service.
*/
@ParameterizedTest @MethodSource("echoUrls") // http and https
public void postFiles(String url) throws IOException {
File thumb = ParseTest.getFile("/htmltests/thumb.jpg");
File html = ParseTest.getFile("/htmltests/large.html");
Document res = Jsoup
.connect(url)
.data("firstname", "Jay")
.data("firstPart", thumb.getName(), Files.newInputStream(thumb.toPath()), "image/jpeg")
.data("secondPart", html.getName(), Files.newInputStream(html.toPath())) // defaults to "application-octetstream";
.data("surname", "Soup")
.post();
assertEquals("4", ihVal("Parts", res));
assertEquals("application/octet-stream", ihVal("Part secondPart ContentType", res));
assertEquals("secondPart", ihVal("Part secondPart Name", res));
assertEquals("large.html", ihVal("Part secondPart Filename", res));
assertEquals("280735", ihVal("Part secondPart Size", res));
assertEquals("image/jpeg", ihVal("Part firstPart ContentType", res));
assertEquals("firstPart", ihVal("Part firstPart Name", res));
assertEquals("thumb.jpg", ihVal("Part firstPart Filename", res));
assertEquals("1052", ihVal("Part firstPart Size", res));
assertEquals("Jay", ihVal("firstname", res));
assertEquals("Soup", ihVal("surname", res));
/*
<tr><th>Part secondPart ContentType</th><td>application/octet-stream</td></tr>
<tr><th>Part secondPart Name</th><td>secondPart</td></tr>
<tr><th>Part secondPart Filename</th><td>google-ipod.html</td></tr>
<tr><th>Part secondPart Size</th><td>43972</td></tr>
<tr><th>Part firstPart ContentType</th><td>image/jpeg</td></tr>
<tr><th>Part firstPart Name</th><td>firstPart</td></tr>
<tr><th>Part firstPart Filename</th><td>thumb.jpg</td></tr>
<tr><th>Part firstPart Size</th><td>1052</td></tr>
*/
}
@Test
public void multipleParsesOkAfterReadFully() throws IOException {
Connection.Response res = Jsoup.connect(echoUrl).execute().readFully();
Document doc = res.parse();
assertTrue(doc.title().contains("Environment"));
Document doc2 = res.parse();
assertTrue(doc2.title().contains("Environment"));
}
@Test
public void multipleParsesOkAfterBufferUp() throws IOException {
Connection.Response res = Jsoup.connect(echoUrl).execute().bufferUp();
Document doc = res.parse();
assertTrue(doc.title().contains("Environment"));
Document doc2 = res.parse();
assertTrue(doc2.title().contains("Environment"));
}
@Test
public void bodyAfterParseThrowsValidationError() {
assertThrows(IllegalArgumentException.class, () -> {
Connection.Response res = Jsoup.connect(echoUrl).execute();
Document doc = res.parse();
String body = res.body();
});
}
@Test
public void bodyAndBytesAvailableBeforeParse() throws IOException {
Connection.Response res = Jsoup.connect(echoUrl).execute();
String body = res.body();
assertTrue(body.contains("Environment"));
byte[] bytes = res.bodyAsBytes();
assertTrue(bytes.length > 100);
Document doc = res.parse();
assertTrue(doc.title().contains("Environment"));
}
@Test
public void parseParseThrowsValidates() {
assertThrows(IllegalArgumentException.class, () -> {
Connection.Response res = Jsoup.connect(echoUrl).execute();
Document doc = res.parse();
assertTrue(doc.title().contains("Environment"));
Document doc2 = res.parse(); // should blow up because the response input stream has been drained
});
}
@Test
public void multiCookieSet() throws IOException {
Connection con = Jsoup
.connect(RedirectServlet.Url)
.data(RedirectServlet.CodeParam, "302")
.data(RedirectServlet.SetCookiesParam, "true")
.data(RedirectServlet.LocationParam, echoUrl);
Connection.Response res = con.execute();
// test cookies set by redirect:
Map<String, String> cookies = res.cookies();
assertEquals("asdfg123", cookies.get("token"));
assertEquals("jhy", cookies.get("uid")); // two uids set, order dependent
// send those cookies into the echo URL by map:
Document doc = Jsoup.connect(echoUrl).cookies(cookies).get();
assertEquals("token=asdfg123; uid=jhy", ihVal("Cookie", doc));
}
@Test public void requestCookiesSurviveRedirect() throws IOException {
// this test makes sure that Request keyval cookies (not in the cookie store) are sent on subsequent redirections,
// when not using the session method
Connection con = Jsoup.connect(RedirectServlet.Url)
.data(RedirectServlet.LocationParam, echoUrl)
.cookie("LetMeIn", "True")
.cookie("DoesItWork", "Yes");
Connection.Response res = con.execute();
assertEquals(0, res.cookies().size()); // were not set by Redir or Echo servlet
Document doc = res.parse();
assertEquals(echoUrl, doc.location());
assertEquals("True", ihVal("Cookie: LetMeIn", doc));
assertEquals("Yes", ihVal("Cookie: DoesItWork", doc));
}
@Test
public void supportsDeflate() throws IOException {
Connection.Response res = Jsoup.connect(DeflateServlet.Url).execute();
assertEquals("deflate", res.header("Content-Encoding"));
Document doc = res.parse();
assertEquals("Hello, World!", doc.selectFirst("p").text());
}
@Test
public void handlesLargerContentLengthParseRead() throws IOException {
// this handles situations where the remote server sets a content length greater than it actually writes
Connection.Response res = Jsoup.connect(InterruptedServlet.Url)
.data(InterruptedServlet.Magnitude, InterruptedServlet.Larger)
.timeout(400)
.execute();
try {
Document document = res.parse();
assertEquals("Something", document.title());
assertEquals(0, document.select("p").size());
} catch (IOException ignored) {
// HttpUrlConnection will read the amount provided and the tests in the try will pass
// HttpClient will throw unexpected EOF during the read, and this will catch it
// Either are OK
}
// current impl, jetty won't write past content length
// todo - find way to trick jetty into writing larger than set header. Take over the stream?
}
@Test
public void handlesWrongContentLengthDuringBufferedRead() throws IOException {
Connection.Response res = Jsoup.connect(InterruptedServlet.Url)
.timeout(400)
.execute();
// this servlet writes max_buffer data, but sets content length to max_buffer/2. So will read up to that.
// previous versions of jetty would allow to write less, and would throw except here
res.bufferUp();
Document doc = res.parse();
assertEquals(0, doc.select("p").size());
}
@Test public void handlesRedirect() throws IOException {
Document doc = Jsoup.connect(RedirectServlet.Url)
.data(RedirectServlet.LocationParam, HelloServlet.Url)
.get();
Element p = doc.selectFirst("p");
assertEquals("Hello, World!", p.text());
assertEquals(HelloServlet.Url, doc.location());
}
@Test public void handlesEmptyRedirect() {
boolean threw = false;
try {
Connection.Response res = Jsoup.connect(RedirectServlet.Url)
.execute();
} catch (IOException e) {
assertTrue(e.getMessage().contains("Too many redirects"));
threw = true;
}
assertTrue(threw);
}
@Test public void doesNotPostFor302() throws IOException {
final Document doc = Jsoup.connect(RedirectServlet.Url)
.data("Hello", "there")
.data(RedirectServlet.LocationParam, EchoServlet.Url)
.post();
assertEquals(EchoServlet.Url, doc.location());
assertEquals("GET", ihVal("Method", doc));
assertNull(ihVal("Hello", doc)); // data not sent
}
@Test public void doesPostFor307() throws IOException {
final Document doc = Jsoup.connect(RedirectServlet.Url)
.data("Hello", "there")
.data(RedirectServlet.LocationParam, EchoServlet.Url)
.data(RedirectServlet.CodeParam, "307")
.post();
assertEquals(EchoServlet.Url, doc.location());
assertEquals("POST", ihVal("Method", doc));
assertEquals("there", ihVal("Hello", doc));
}
@Test public void getUtf8Bom() throws IOException {
Connection con = Jsoup.connect(FileServlet.urlTo("/bomtests/bom_utf8.html"));
Document doc = con.get();
assertEquals("UTF-8", con.response().charset());
assertEquals("OK", doc.title());
}
@Test public void streamerGetUtf8Bom() throws IOException {
Connection con = Jsoup.connect(FileServlet.urlTo("/bomtests/bom_utf8.html"));
Document doc = con.execute().streamParser().complete();
assertEquals("UTF-8", con.response().charset());
assertEquals("OK", doc.title());
}
@Test
public void testBinaryContentTypeThrowsException() throws IOException {
Connection con = Jsoup.connect(FileServlet.urlTo("/htmltests/thumb.jpg"));
con.data(FileServlet.ContentTypeParam, "image/jpeg");
boolean threw = false;
try {
con.execute();
Document doc = con.response().parse();
} catch (UnsupportedMimeTypeException e) {
threw = true;
assertEquals("Unhandled content type. Must be text/*, */xml, or */*+xml", e.getMessage());
}
assertTrue(threw);
}
@Test public void testParseRss() throws IOException {
// test that we switch automatically to xml, and we support application/rss+xml
Connection con = Jsoup.connect(FileServlet.urlTo("/htmltests/test-rss.xml"));
con.data(FileServlet.ContentTypeParam, "application/rss+xml");
Document doc = con.get();
Element title = doc.selectFirst("title");
assertNotNull(title);
assertEquals("jsoup RSS news", title.text());
assertEquals("channel", title.parent().nodeName());
assertEquals("", doc.title()); // the document title is unset, this tag is channel>title, not html>head>title
assertEquals(3, doc.select("link").size());
assertEquals("application/rss+xml", con.response().contentType());
assertTrue(doc.parser().getTreeBuilder() instanceof XmlTreeBuilder);
assertEquals(Document.OutputSettings.Syntax.xml, doc.outputSettings().syntax());
}
@Test
public void testSupplyParserToConnection() throws IOException {
String xmlUrl = FileServlet.urlTo("/htmltests/xml-test.xml");
// parse with both xml and html parser, ensure different
Document xmlDoc = Jsoup.connect(xmlUrl).parser(Parser.xmlParser()).get();
Document htmlDoc = Jsoup.connect(xmlUrl).parser(Parser.htmlParser()).get();
Document autoXmlDoc = Jsoup.connect(xmlUrl).get(); // check connection auto detects xml, uses xml parser
assertEquals("<doc><val>One<val>Two</val>Three</val></doc>",
TextUtil.stripNewlines(xmlDoc.html()));
assertNotEquals(htmlDoc, xmlDoc);
assertFalse(htmlDoc.hasSameValue(xmlDoc));
assertTrue(xmlDoc.hasSameValue(autoXmlDoc));
assertEquals(1, htmlDoc.select("head").size()); // html parser normalises
assertEquals(0, xmlDoc.select("head").size()); // xml parser does not
assertEquals(0, autoXmlDoc.select("head").size()); // xml parser does not
}
@Test public void imageXmlMimeType() throws IOException {
// test that we switch to XML, and that we support image/svg+xml
String mimetype = "image/svg+xml";
Connection con = Jsoup.connect(FileServlet.urlTo("/htmltests/osi-logo.svg"))
.data(FileServlet.ContentTypeParam, mimetype);
Document doc = con.get();
assertEquals(mimetype, con.response().contentType());
assertTrue(doc.parser().getTreeBuilder() instanceof XmlTreeBuilder);
assertEquals(Document.OutputSettings.Syntax.xml, doc.outputSettings().syntax());
Node firstChild = doc.firstChild();
XmlDeclaration decl = (XmlDeclaration) firstChild;
assertEquals("no", decl.attr("standalone"));
Element svg = doc.expectFirst("svg");
Element flowRoot = svg.expectFirst("flowRoot");
assertEquals("flowRoot", flowRoot.tagName());
assertEquals("preserve", flowRoot.attr("xml:space"));
}
@Test
public void canFetchBinaryAsBytes() throws IOException {
String path = "/htmltests/thumb.jpg";
int actualSize = 1052;
Connection.Response res = Jsoup.connect(FileServlet.urlTo(path))
.data(FileServlet.ContentTypeParam, "image/jpeg")
.ignoreContentType(true)
.execute();
byte[] resBytes = res.bodyAsBytes();
assertEquals(actualSize, resBytes.length);
// compare the content of the file and the bytes:
Path filePath = ParseTest.getPath(path);
byte[] fileBytes = Files.readAllBytes(filePath);
assertEquals(actualSize, fileBytes.length);
assertArrayEquals(fileBytes, resBytes);
}
@Test
public void handlesUnknownEscapesAcrossBuffer() throws IOException {
String localPath = "/htmltests/escapes-across-buffer.html";
String localUrl = FileServlet.urlTo(localPath);
Document docFromLocalServer = Jsoup.connect(localUrl).get();
Document docFromFileRead = Jsoup.parse(ParseTest.getFile(localPath), "UTF-8");
String text = docFromLocalServer.body().text();
assertEquals(14766, text.length());
assertEquals(text, docFromLocalServer.body().text());
assertEquals(text, docFromFileRead.body().text());
}
/**
* Test fetching a form, and submitting it with a file attached.
*/
@Test
public void postHtmlFile() throws IOException {
Document index = Jsoup.connect(FileServlet.urlTo("/htmltests/upload-form.html")).get();
List<FormElement> forms = index.select("[name=tidy]").forms();
assertEquals(1, forms.size());
FormElement form = forms.get(0);
Connection post = form.submit();
File uploadFile = ParseTest.getFile("/htmltests/large.html");
FileInputStream stream = new FileInputStream(uploadFile);
Connection.KeyVal fileData = post.data("_file");
assertNotNull(fileData);
fileData.value("check.html");
fileData.inputStream(stream);
Connection.Response res;
try {
res = post.execute();
} finally {
stream.close();
}
Document doc = res.parse();
assertEquals(ihVal("Method", doc), "POST"); // from form action
assertEquals(ihVal("Part _file Filename", doc), "check.html");
assertEquals(ihVal("Part _file Name", doc), "_file");
assertEquals(ihVal("_function", doc), "tidy");
}
@Test
public void fetchHandlesXml() throws IOException {
String[] types = {"text/xml", "application/xml", "application/rss+xml", "application/xhtml+xml"};
for (String type : types) {
fetchHandlesXml(type);
}
}
void fetchHandlesXml(String contentType) throws IOException {
// should auto-detect xml and use XML parser, unless explicitly requested the html parser
String xmlUrl = FileServlet.urlTo("/htmltests/xml-test.xml");
Connection con = Jsoup.connect(xmlUrl);
con.data(FileServlet.ContentTypeParam, contentType);
Document doc = con.get();
Connection.Request req = con.request();
assertTrue(req.parser().getTreeBuilder() instanceof XmlTreeBuilder);
assertEquals("<doc><val>One<val>Two</val>Three</val></doc>\n", doc.outerHtml());
assertEquals(con.response().contentType(), contentType);
}
@Test
public void fetchHandlesXmlAsHtmlWhenParserSet() throws IOException {
// should auto-detect xml and use XML parser, unless explicitly requested the html parser
String xmlUrl = FileServlet.urlTo("/htmltests/xml-test.xml");
Connection con = Jsoup.connect(xmlUrl).parser(Parser.htmlParser()); // which will also use the pretty printer by default
con.data(FileServlet.ContentTypeParam, "application/xml");
Document doc = con.get();
Connection.Request req = con.request();
assertTrue(req.parser().getTreeBuilder() instanceof HtmlTreeBuilder);
assertEquals("<html>\n <head></head>\n <body>\n <doc>\n <val>\n One<val>Two</val>Three\n </val>\n </doc>\n </body>\n</html>", doc.outerHtml());
}
@Test
public void combinesSameHeadersWithComma() throws IOException {
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
Connection con = Jsoup.connect(echoUrl);
con.get();
Connection.Response res = con.response();
assertEquals("text/html;charset=utf-8", res.header("Content-Type"));
assertEquals("no-cache, no-store", res.header("Cache-Control"));
List<String> header = res.headers("Cache-Control");
assertEquals(2, header.size());
assertEquals("no-cache", header.get(0));
assertEquals("no-store", header.get(1));
}
@Test
public void sendHeadRequest() throws IOException {
String url = FileServlet.urlTo("/htmltests/xml-test.xml");
Connection con = Jsoup.connect(url)
.method(Connection.Method.HEAD)
.data(FileServlet.ContentTypeParam, "text/xml");
final Connection.Response response = con.execute();
assertEquals("text/xml", response.header("Content-Type"));
assertEquals("", response.body()); // head ought to have no body
Document doc = response.parse();
assertEquals("", doc.text());
}
@Test
public void fetchToW3c() throws IOException {
String url = FileServlet.urlTo("/htmltests/upload-form.html");
Document doc = Jsoup.connect(url).get();
W3CDom dom = new W3CDom();
org.w3c.dom.Document wDoc = dom.fromJsoup(doc);
assertEquals(url, wDoc.getDocumentURI());
String html = dom.asString(wDoc);
assertTrue(html.contains("Upload"));
}
@Test
public void baseHrefCorrectAfterHttpEquiv() throws IOException {
// https://github.com/jhy/jsoup/issues/440
Connection.Response res = Jsoup.connect(FileServlet.urlTo("/htmltests/charset-base.html")).execute();
Document doc = res.parse();
assertEquals("http://example.com/foo.jpg", doc.select("img").first().absUrl("src"));
}
@Test
public void maxBodySize() throws IOException {
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
Connection.Response defaultRes = Jsoup.connect(url).execute();
Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops
Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops
Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop
Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();
int actualDocText = LargeDocTextLen;
assertEquals(actualDocText, defaultRes.parse().text().length());
assertEquals(49165, smallRes.parse().text().length());
assertEquals(196577, mediumRes.parse().text().length());
assertEquals(actualDocText, largeRes.parse().text().length());
assertEquals(actualDocText, unlimitedRes.parse().text().length());
}
@Test public void repeatable() throws IOException {
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
Connection con = Jsoup.connect(url).parser(Parser.xmlParser());
Document doc1 = con.get();
Document doc2 = con.get();
assertEquals("Large HTML", doc1.title());
assertEquals("Large HTML", doc2.title());
}
@Test
public void maxBodySizeInReadToByteBuffer() throws IOException {
// https://github.com/jhy/jsoup/issues/1774
// when calling readToByteBuffer, contents were not buffered up
String url = FileServlet.urlTo("/htmltests/large.html"); // 280 K
Connection.Response defaultRes = Jsoup.connect(url).execute();
Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops
Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops
Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop
Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();
int actualDocText = 280735;
assertEquals(actualDocText, defaultRes.body().length());
assertEquals(50 * 1024, smallRes.body().length());
assertEquals(200 * 1024, mediumRes.body().length());
assertEquals(actualDocText, largeRes.body().length());
assertEquals(actualDocText, unlimitedRes.body().length());
assertEquals(actualDocText, defaultRes.readBody().length());
assertEquals(50 * 1024, smallRes.readBody().length());
assertEquals(200 * 1024, mediumRes.readBody().length());
assertEquals(actualDocText, largeRes.readBody().length());
assertEquals(actualDocText, unlimitedRes.readBody().length());
}
@Test void formLoginFlow() throws IOException {
String echoUrl = EchoServlet.Url;
String cookieUrl = CookieServlet.Url;
String startUrl = FileServlet.urlTo("/htmltests/form-tests.html");
Document loginDoc = Jsoup.connect(startUrl).get();
FormElement form = loginDoc.expectForm("#login");
assertNotNull(form);
form.expectFirst("[name=username]").val("admin");
form.expectFirst("[name=password]").val("Netscape engineers are weenies!");
// post it- should go to Cookie then bounce to Echo
Connection submit = form.submit();
assertEquals(Connection.Method.POST, submit.request().method());
Connection.Response postRes = submit.execute();
assertEquals(echoUrl, postRes.url().toExternalForm());
assertEquals(Connection.Method.GET, postRes.method());
Document resultDoc = postRes.parse();
assertEquals("One=EchoServlet; One=Root", ihVal("Cookie", resultDoc));
// should be no form data sent to the echo redirect
assertEquals("", ihVal("Query String", resultDoc));
// new request to echo, should not have form data, but should have cookies from implicit session
Document newEcho = submit.newRequest(echoUrl).get();
assertEquals("One=EchoServlet; One=Root", ihVal("Cookie", newEcho));
assertEquals("", ihVal("Query String", newEcho));
Document cookieDoc = submit.newRequest(cookieUrl).get();
assertEquals("CookieServlet", ihVal("One", cookieDoc)); // different cookie path
}
@Test void formLoginFlow2() throws IOException {
String echoUrl = EchoServlet.Url;
String cookieUrl = CookieServlet.Url;
String startUrl = FileServlet.urlTo("/htmltests/form-tests.html");
Connection session = Jsoup.newSession();
Document loginDoc = session.newRequest(startUrl).get();
FormElement form = loginDoc.expectForm("#login2");
assertNotNull(form);
String username = "admin";
form.expectFirst("[name=username]").val(username);
String password = "Netscape engineers are weenies!";
form.expectFirst("[name=password]").val(password);
Connection submit = form.submit();
assertEquals(username, submit.data("username").value());
assertEquals(password, submit.data("password").value());
Connection.Response postRes = submit.execute();
assertEquals(cookieUrl, postRes.url().toExternalForm());
assertEquals(Connection.Method.POST, postRes.method());
Document resultDoc = postRes.parse();
Document echo2 = resultDoc.connection().newRequest(echoUrl).get();
assertEquals("", ihVal("Query String", echo2)); // should not re-send the data
assertEquals("One=EchoServlet; One=Root", ihVal("Cookie", echo2));
}
@Test void preservesUrlFragment() throws IOException {
// confirms https://github.com/jhy/jsoup/issues/1686
String url = EchoServlet.Url + "#fragment";
Document doc = Jsoup.connect(url).get();
assertEquals(url, doc.location());
}
@Test void fetchUnicodeUrl() throws IOException {
String url = EchoServlet.Url + "/✔/?鍵=値";
Document doc = Jsoup.connect(url).get();
assertEquals("/✔/", ihVal("Path Info", doc));
assertEquals("%E9%8D%B5=%E5%80%A4", ihVal("Query String", doc));
assertEquals("鍵=値", URLDecoder.decode(ihVal("Query String", doc), DataUtil.UTF_8.name()));
}
@Test void willEscapePathInRedirect() throws IOException {
String append = "/Foo{bar}<>%/";
String url = EchoServlet.Url + append;
Document doc = Jsoup
.connect(RedirectServlet.Url)
.data(RedirectServlet.LocationParam, url)
.get();
String path = ihVal("Path Info", doc);
assertEquals(append, path);
assertEquals("/EchoServlet/Foo%7Bbar%7D%3C%3E%25/", ihVal("Request URI", doc));
}
/**
Provides HTTP and HTTPS EchoServlet URLs
*/
private static Stream<String> echoUrls() {
return Stream.of(EchoServlet.Url, EchoServlet.TlsUrl);
}
@ParameterizedTest @MethodSource("echoUrls")
void failsIfNotAuthenticated(String url) throws IOException {
String password = AuthFilter.newServerPassword(); // we don't send it, but ensures cache won't hit
Connection.Response res = Jsoup.connect(url)
.header(AuthFilter.WantsServerAuthentication, "1")
.ignoreHttpErrors(true)
.execute();
assertEquals(401, res.statusCode());
}
@ParameterizedTest @MethodSource("echoUrls")
void canAuthenticate(String url) throws IOException {
AtomicInteger count = new AtomicInteger(0);
String password = AuthFilter.newServerPassword();
Connection.Response res = Jsoup.connect(url)
.header(AuthFilter.WantsServerAuthentication, "1")
.auth(ctx -> {
count.incrementAndGet();
assertEquals(Authenticator.RequestorType.SERVER, ctx.type());
assertEquals("localhost", ctx.url().getHost());
assertEquals(AuthFilter.ServerRealm, ctx.realm());
return ctx.credentials(AuthFilter.ServerUser, password);
})
.execute();
assertEquals(1, count.get());
Document doc = res.parse();
assertTrue(ihVal("Authorization", doc).startsWith("Basic ")); // tests we set the auth header
}
@ParameterizedTest @MethodSource("echoUrls")
void incorrectAuth(String url) throws IOException {
Connection session = Jsoup.newSession()
.header(AuthFilter.WantsServerAuthentication, "1")
.ignoreHttpErrors(true);
String password = AuthFilter.newServerPassword();
int code = session.newRequest(url).execute().statusCode(); // no auth sent
assertEquals(HttpServletResponse.SC_UNAUTHORIZED, code);
AtomicInteger count = new AtomicInteger(0);
try {
Connection.Response res = session.newRequest(url)
.auth(ctx -> {
count.incrementAndGet();
return ctx.credentials(AuthFilter.ServerUser, password + "wrong"); // incorrect
})
.execute();
assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.statusCode());
} catch (IOException e) {
assertEquals("No credentials provided", e.getMessage());
// In HttpClient, will throw IOE if our password delegate stops providing credentials after too many attempts. So we'll get this error.
// In HttpUrlConnection, which would otherwise try 20 times, when the auth stops providing it will cascade to the underyling 401 response (which seems a better path IMO)
}
assertEquals(MaxAttempts, count.get());
AtomicInteger successCount = new AtomicInteger(0);
Connection.Response successRes = session.newRequest(url)
.auth(ctx -> {
successCount.incrementAndGet();
return ctx.credentials(AuthFilter.ServerUser, password); // correct
})
.execute();
assertEquals(1, successCount.get());
assertEquals(HttpServletResponse.SC_OK, successRes.statusCode());
}
// proxy connection tests are in ProxyTest
@ParameterizedTest
@ValueSource(strings = {
"/htmltests/large.html",
"/htmltests/large.html?" + FileServlet.SuppressContentLength
})
void progressListener(String path) throws IOException {
String url = FileServlet.urlTo(path);
boolean knownContentLength = !url.contains(FileServlet.SuppressContentLength);
AtomicBoolean seenProgress = new AtomicBoolean(false);
AtomicBoolean completed = new AtomicBoolean(false);
AtomicInteger numProgress = new AtomicInteger();
Connection con = Jsoup.connect(url).onResponseProgress((processed, total, percent, response) -> {
//System.out.println("Processed: " + processed + " of " + total + " (" + percent + "%)");
if (!seenProgress.get()) {
seenProgress.set(true);
assertEquals(0, processed);
assertEquals(knownContentLength ? LargeDocFileLen : -1, total);
assertEquals(0.0f, percent);
assertEquals(200, response.statusCode());
String contentLength = response.header("Content-Length");
if (knownContentLength) {
assertNotNull(contentLength);
assertEquals(String.valueOf(LargeDocFileLen), contentLength);
} else {
assertNull(contentLength);
}
assertEquals(url, response.url().toExternalForm());
}
numProgress.getAndIncrement();
if (percent == 100.0f) {
// even if the content-length is not set, we get 100% when the read is completed
completed.set(true);
assertEquals(LargeDocFileLen, processed);
}
});
Document document = con.get();
assertTrue(seenProgress.get());
assertTrue(completed.get());
// should expect to see events relative to how large the buffer is.
int expected = LargeDocFileLen / 8192;
int num = numProgress.get();
// debug log if not in those ranges:
if (num < expected * 0.75 || num > expected * 2.5) {
System.err.println("Expected: " + expected + ", got: " + num);
}
assertTrue(num > expected * 0.75);
assertTrue(num < expected * 2.5);
// check the document works
assertEquals(LargeDocTextLen, document.text().length());
}
}
================================================
FILE: src/test/java/org/jsoup/integration/FuzzFixesIT.java
================================================
package org.jsoup.integration;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import java.io.File;
import java.io.IOException;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
Tests fixes for issues raised by the <a href="https://oss-fuzz.com/testcases?project=jsoup">OSS Fuzz project</a>. As
some of these are timeout tests - run each file 100 times and ensure under time.
*/
public class FuzzFixesIT {
static int numIters = 50;
static int timeout = 30; // external fuzzer is set to 60 for 100 runs
static File testDir = ParseTest.getFile("/fuzztests/");
private static Stream<File> testFiles() {
File[] files = testDir.listFiles();
assertNotNull(files);
assertTrue(files.length > 10);
return Stream.of(files);
}
@Disabled // disabled, as these soak up build time and the outcome oughtn't change unless we are refactoring the tree builders. manually execute as desired.
@ParameterizedTest
@MethodSource("testFiles")
void testHtmlParse(File file) throws IOException {
long startTime = System.currentTimeMillis();
long completeBy = startTime + timeout * 1000L;
for (int i = 0; i < numIters; i++) {
Document doc = Jsoup.parse(file, "UTF-8", "https://example.com/");
assertNotNull(doc);
if (System.currentTimeMillis() > completeBy)
Assertions.fail(String.format("Timeout: only completed %d iters of [%s] in %d seconds", i, file.getName(), timeout));
}
}
@Disabled // disabled, as these soak up build time and the outcome oughtn't change unless we are refactoring the tree builders. manually execute as desired.
@ParameterizedTest
@MethodSource("testFiles")
void testXmlParse(File file) throws IOException {
long startTime = System.currentTimeMillis();
long completeBy = startTime + timeout * 1000L;
for (int i = 0; i < numIters; i++) {
Document doc = Jsoup.parse(file, "UTF-8", "https://example.com/", Parser.xmlParser());
assertNotNull(doc);
if (System.currentTimeMillis() > completeBy)
Assertions.fail(String.format("Timeout: only completed %d iters of [%s] in %d seconds", i, file.getName(), timeout));
}
}
}
================================================
FILE: src/test/java/org/jsoup/integration/FuzzFixesTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import java.io.File;
import java.io.IOException;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
Tests fixes for issues raised by the OSS Fuzz project @ https://oss-fuzz.com/testcases?project=jsoup. Contains inline
string cases causing exceptions. Timeout tests are in FuzzFixesIT.
*/
public class FuzzFixesTest {
private static Stream<File> testFiles() {
File[] files = FuzzFixesIT.testDir.listFiles();
assertNotNull(files);
assertTrue(files.length > 10);
return Stream.of(files);
}
@Test
public void blankAbsAttr() {
// https://github.com/jhy/jsoup/issues/1541
String html = "b<bodY abs: abs:abs: abs:abs:abs>";
Document doc = Jsoup.parse(html);
assertNotNull(doc);
}
@Test
public void bookmark() {
// https://github.com/jhy/jsoup/issues/1576
String html = "<?a<U<P<A ";
Document doc = Jsoup.parse(html);
assertNotNull(doc);
Document xmlDoc = Parser.xmlParser().parseInput(html, "");
assertNotNull(xmlDoc);
}
@Test void fragment() {
Parser.htmlParser().parseFragmentInput("<frameset>>l\u0000<\u0000<ditl>\u0000< \\", new Element("colgroup"), "");
}
@ParameterizedTest
@MethodSource("testFiles")
void testHtmlParse(File file) throws IOException {
Document doc = Jsoup.parse(file, "UTF-8", "https://example.com/");
assertNotNull(doc);
doc = Jsoup.parse(file, "UTF-8", ""); // no base href attr; so same as a parse(string), which can have subtly different semantics
assertNotNull(doc);
}
@ParameterizedTest
@MethodSource("testFiles")
void testHtmlFragmentParse(File file) throws IOException {
String html = ParseTest.getFileAsString(file);
Document doc = Jsoup.parseBodyFragment(html);
assertNotNull(doc);
}
@ParameterizedTest
@MethodSource("testFiles")
void testXmlParse(File file) throws IOException {
Document doc = Jsoup.parse(file, "UTF-8", "https://example.com/", Parser.xmlParser());
assertNotNull(doc);
doc = Jsoup.parse(file, "UTF-8", "", Parser.xmlParser()); // no base href attr
}
}
================================================
FILE: src/test/java/org/jsoup/integration/ParseTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Jsoup;
import org.jsoup.helper.DataUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.ParseErrorList;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.GZIPInputStream;
import static org.junit.jupiter.api.Assertions.*;
/**
* Integration test: parses from real-world example HTML.
*
* @author Jonathan Hedley, jonathan@hedley.net
*/
public class ParseTest {
@Test
public void testHtml5Charset() throws IOException {
// test that <meta charset="gb2312"> works
File in = getFile("/htmltests/meta-charset-1.html");
Document doc = Jsoup.parse(in, null, "http://example.com/"); //gb2312, has html5 <meta charset>
assertEquals("新", doc.text());
assertEquals("GB2312", doc.outputSettings().charset().displayName());
// double check, no charset, falls back to utf8 which is incorrect
in = getFile("/htmltests/meta-charset-2.html"); //
doc = Jsoup.parse(in, null, "http://example.com"); // gb2312, no charset
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
assertNotEquals("新", doc.text());
// confirm fallback to utf8
in = getFile("/htmltests/meta-charset-3.html");
doc = Jsoup.parse(in, null, "http://example.com/"); // utf8, no charset
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
assertEquals("新", doc.text());
}
@Test
public void testBrokenHtml5CharsetWithASingleDoubleQuote() throws IOException {
InputStream in = inputStreamFrom("<html>\n" +
"<head><meta charset=UTF-8\"></head>\n" +
"<body></body>\n" +
"</html>");
Document doc = Jsoup.parse(in, null, "http://example.com/");
assertEquals("UTF-8", doc.outputSettings().charset().displayName());
}
@Test
public void testLowercaseUtf8Charset() throws IOException {
File in = getFile("/htmltests/lowercase-charset-test.html");
Document doc = Jsoup.parse(in, null);
Element form = doc.select("#form").first();
assertEquals(2, form.children().size());
assertEquals("UTF-8", doc.outputSettings().charset().name());
}
@Test
public void testXwiki() throws IOException {
// https://github.com/jhy/jsoup/issues/1324
// this tests that when in CharacterReader we hit a buffer while marked, we preserve the mark when buffered up and can rewind
File in = getFile("/htmltests/xwiki-1324.html.gz");
Document doc = Jsoup.parse(in, null, "https://localhost/");
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
// was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so
// updated to preserve the mark.
String wantHtml = "<a class=\"list-group-item\" data-id=\"userdirectory\" href=\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin§ion=userdirectory\" title=\"Customize the user directory live table.\">User Directory</a>";
assertEquals(wantHtml, doc.select("[data-id=userdirectory]").outerHtml());
}
@Test
public void testXwikiExpanded() throws IOException {
// https://github.com/jhy/jsoup/issues/1324
// this tests that if there is a huge illegal character reference, we can get through a buffer and rewind, and still catch that it's an invalid refence,
// and the parse tree is correct.
File in = getFile("/htmltests/xwiki-edit.html.gz");
Parser parser = Parser.htmlParser();
Document doc = Jsoup.parse(new GZIPInputStream(new FileInputStream(in)), "UTF-8", "https://localhost/", parser.setTrackErrors(100));
ParseErrorList errors = parser.getErrors();
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
assertEquals(0, errors.size()); // not an invalid reference because did not look legit
// was getting busted at =userdirectory, because it hit the bufferup point but the mark was then lost. so
// updated to preserve the mark.
String wantHtml = "<a class=\"list-group-item\" data-id=\"userdirectory\" href=\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin&RIGHTHERERIGHTHERERIGHTHERERIGHTHERE";
assertTrue(doc.select("[data-id=userdirectory]").outerHtml().startsWith(wantHtml));
}
@Test public void testWikiExpandedFromString() throws IOException {
File in = getFile("/htmltests/xwiki-edit.html.gz");
String html = getFileAsString(in);
Document doc = Jsoup.parse(html);
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
String wantHtml = "<a class=\"list-group-item\" data-id=\"userdirectory\" href=\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin&RIGHTHERERIGHTHERERIGHTHERERIGHTHERE";
assertTrue(doc.select("[data-id=userdirectory]").outerHtml().startsWith(wantHtml));
}
@Test public void testWikiFromString() throws IOException {
File in = getFile("/htmltests/xwiki-1324.html.gz");
String html = getFileAsString(in);
Document doc = Jsoup.parse(html);
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
String wantHtml = "<a class=\"list-group-item\" data-id=\"userdirectory\" href=\"/xwiki/bin/admin/XWiki/XWikiPreferences?editor=globaladmin§ion=userdirectory\" title=\"Customize the user directory live table.\">User Directory</a>";
assertEquals(wantHtml, doc.select("[data-id=userdirectory]").outerHtml());
}
@Test public void testFileParseNoCharsetMethod() throws IOException {
File in = getFile("/htmltests/xwiki-1324.html.gz");
Document doc = Jsoup.parse(in);
assertEquals("XWiki Jetty HSQLDB 12.1-SNAPSHOT", doc.select("#xwikiplatformversion").text());
}
public static File getFile(String resourceName) {
try {
URL resource = ParseTest.class.getResource(resourceName);
return resource != null ? new File(resource.toURI()) : new File("/404");
} catch (URISyntaxException e) {
throw new IllegalStateException(e);
}
}
public static Path getPath(String resourceName) {
try {
URL resource = ParseTest.class.getResource(resourceName);
return resource != null ? Paths.get(resource.toURI()) : Paths.get("/404");
} catch (URISyntaxException e) {
throw new IllegalStateException(e);
}
}
public static InputStream inputStreamFrom(String s) {
return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
}
public static String getFileAsString(File file) throws IOException {
byte[] bytes;
if (file.getName().endsWith(".gz")) {
InputStream stream = new GZIPInputStream(new FileInputStream(file));
ByteBuffer byteBuffer = DataUtil.readToByteBuffer(stream, 0);
bytes = new byte[byteBuffer.limit()];
System.arraycopy(byteBuffer.array(), 0, bytes, 0, byteBuffer.limit());
} else {
bytes = Files.readAllBytes(file.toPath());
}
return new String(bytes);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/ProxyTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.integration.servlets.AuthFilter;
import org.jsoup.integration.servlets.EchoServlet;
import org.jsoup.integration.servlets.FileServlet;
import org.jsoup.integration.servlets.HelloServlet;
import org.jsoup.integration.servlets.ProxyServlet;
import org.jsoup.integration.servlets.RedirectServlet;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import static org.jsoup.helper.AuthenticationHandlerTest.MaxAttempts;
import static org.jsoup.integration.ConnectTest.ihVal;
import static org.junit.jupiter.api.Assertions.*;
/**
Tests Jsoup.connect proxy support */
public class ProxyTest {
private static String echoUrl;
private static TestServer.ProxySettings proxy;
@BeforeAll
public static void setUp() {
echoUrl = EchoServlet.Url;
proxy = ProxyServlet.ProxySettings;
}
@ParameterizedTest @MethodSource("helloUrls")
void fetchViaProxy(String url) throws IOException {
Connection con = Jsoup.connect(url)
.proxy(proxy.hostname, proxy.port);
Connection.Response res = con.execute();
if (url.startsWith("http:/")) assertVia(res); // HTTPS CONNECT won't have Via
Document doc = res.parse();
Element p = doc.expectFirst("p");
assertEquals("Hello, World!", p.text());
}
private static Stream<String> helloUrls() {
return Stream.of(HelloServlet.Url, HelloServlet.TlsUrl);
}
private static Stream<String> echoUrls() {
return Stream.of(EchoServlet.Url, EchoServlet.TlsUrl);
}
private static void assertVia(Connection.Response res) {
assertEquals(res.header("Via"), ProxyServlet.Via);
}
@Test void redirectViaProxy() throws IOException {
Connection.Response res = Jsoup
.connect(RedirectServlet.Url)
.data(RedirectServlet.LocationParam, echoUrl)
.header("Random-Header-name", "hello")
.proxy(proxy.hostname, proxy.port)
.execute();
assertVia(res);
Document doc = res.parse();
assertEquals(echoUrl, doc.location());
assertEquals("hello", ihVal("Random-Header-name", doc));
assertVia(res);
}
@Test void proxyForSession() throws IOException {
Connection session = Jsoup.newSession().proxy(proxy.hostname, proxy.port);
Connection.Response medRes = session.newRequest(FileServlet.urlTo("/htmltests/medium.html")).execute();
Connection.Response largeRes = session.newRequest(FileServlet.urlTo("/htmltests/large.html")).execute();
assertVia(medRes);
assertVia(largeRes);
assertEquals("Medium HTML", medRes.parse().title());
assertEquals("Large HTML", largeRes.parse().title());
Connection.Response smedRes = session.newRequest(FileServlet.tlsUrlTo("/htmltests/medium.html")).execute();
Connection.Response slargeRes = session.newRequest(FileServlet.tlsUrlTo("/htmltests/large.html")).execute();
assertEquals("Medium HTML", smedRes.parse().title());
assertEquals("Large HTML", slargeRes.parse().title());
}
@ParameterizedTest @MethodSource("echoUrls")
void canAuthenticateToProxy(String url) throws IOException {
int closed = TestServer.closeAuthedProxyConnections(); // reset any existing authed connections from previous tests, so we can test the auth flow
// the proxy wants auth, but not the server. HTTP and HTTPS, so tests direct proxy and CONNECT
Connection session = Jsoup.newSession()
.proxy(proxy.hostname, proxy.authedPort)
.ignoreHttpErrors(true)
.ignoreContentType(true); // ignore content type, as error served may not have a content type
String password = AuthFilter.newProxyPassword();
// fail first
try {
Connection.Response execute = session.newRequest(url)
.execute();
int code = execute.statusCode(); // no auth sent
assertEquals(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED, code);
} catch (IOException e) {
assertAuthRequiredException(e);
}
try {
AtomicInteger count = new AtomicInteger(0);
Connection.Response res = session.newRequest(url)
.auth(ctx -> {
count.incrementAndGet();
return ctx.credentials(AuthFilter.ProxyUser, password + "wrong"); // incorrect
})
.execute();
assertEquals(MaxAttempts, count.get());
assertEquals(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED, res.statusCode());
} catch (IOException e) {
assertAuthRequiredException(e);
}
AtomicInteger successCount = new AtomicInteger(0);
Connection.Response successRes = session.newRequest(url)
.auth(ctx -> {
successCount.incrementAndGet();
return ctx.credentials(AuthFilter.ProxyUser, password); // correct
})
.execute();
assertEquals(1, successCount.get());
assertEquals(HttpServletResponse.SC_OK, successRes.statusCode());
}
static void assertAuthRequiredException(IOException e) {
// in CONNECT (for the HTTPS url), URLConnection will throw the proxy connect as a Stringly typed IO exception - "Unable to tunnel through proxy. Proxy returns "HTTP/1.1 407 Proxy Authentication Required"". (Not a response code)
// Alternatively, some platforms (?) will report: "No credentials provided"
String err = e.getMessage();
if (!(err.contains("407") || err.contains("No credentials provided") || err.contains("exch.exchImpl"))) {
// https://github.com/jhy/jsoup/pull/2403 - Ubuntu Azul 25 throws `Cannot invoke "jdk.internal.net.http.ExchangeImpl.cancel(java.io.IOException)" because "exch.exchImpl" is null` here but is just from cancelling the 407 req
System.err.println("Not a 407 exception? " + e.getClass());
e.printStackTrace(System.err);
fail("Expected 407 Proxy Authentication Required, got: " + err);
}
}
@ParameterizedTest @MethodSource("echoUrls")
void canAuthToProxyAndServer(String url) throws IOException {
String serverPassword = AuthFilter.newServerPassword();
String proxyPassword = AuthFilter.newProxyPassword();
AtomicInteger count = new AtomicInteger(0);
Connection session = Jsoup.newSession() // both proxy and server will want auth
.proxy(proxy.hostname, proxy.authedPort)
.header(AuthFilter.WantsServerAuthentication, "1")
.auth(auth -> {
count.incrementAndGet();
if (auth.isServer()) {
assertEquals(url, auth.url().toString());
assertEquals(AuthFilter.ServerRealm, auth.realm());
return auth.credentials(AuthFilter.ServerUser, serverPassword);
} else {
assertTrue(auth.isProxy());
return auth.credentials(AuthFilter.ProxyUser, proxyPassword);
}
});
Connection.Response res = session.newRequest(url).execute();
assertEquals(200, res.statusCode());
assertEquals(2, count.get()); // hit server and proxy auth stages
assertEquals("Webserver Environment Variables", res.parse().title());
}
}
================================================
FILE: src/test/java/org/jsoup/integration/SafelistExtensionTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Jsoup;
import org.jsoup.TextUtil;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Element;
import org.jsoup.safety.Safelist;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
Check that we can extend Safelist methods
*/
public class SafelistExtensionTest {
@Test public void canCustomizeSafeTests() {
OpenSafelist openSafelist = new OpenSafelist(Safelist.relaxed());
Safelist safelist = Safelist.relaxed();
String html = "<p><opentag openattr>Hello</opentag></p>";
String openClean = Jsoup.clean(html, openSafelist);
String clean = Jsoup.clean(html, safelist);
assertEquals("<p><opentag openattr=\"\">Hello</opentag></p>", TextUtil.stripNewlines(openClean));
assertEquals("<p>Hello</p>", clean);
}
// passes tags and attributes starting with "open"
private static class OpenSafelist extends Safelist {
public OpenSafelist(Safelist safelist) {
super(safelist);
}
@Override
public boolean isSafeAttribute(String tagName, Element el, Attribute attr) {
if (attr.getKey().startsWith("open"))
return true;
return super.isSafeAttribute(tagName, el, attr);
}
@Override
public boolean isSafeTag(String tag) {
if (tag.startsWith("open"))
return true;
return super.isSafeTag(tag);
}
}
}
================================================
FILE: src/test/java/org/jsoup/integration/SessionIT.java
================================================
package org.jsoup.integration;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.integration.servlets.FileServlet;
import org.jsoup.integration.servlets.SlowRider;
import org.jsoup.nodes.Document;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.*;
/** Integration tests to test longer running Connection */
public class SessionIT {
@BeforeAll
public static void setUp() {
TestServer.start();
}
@Test
public void multiThread() throws InterruptedException {
int numThreads = 20;
int numThreadLoops = 5;
String[] urls = {
FileServlet.urlTo("/htmltests/medium.html"),
FileServlet.urlTo("/htmltests/upload-form.html"),
FileServlet.urlTo("/htmltests/comments.html"),
FileServlet.urlTo("/htmltests/large.html"),
};
String[] titles = {
"Medium HTML",
"Upload Form Test",
"A Certain Kind of Test",
"Large HTML"
};
ThreadCatcher catcher = new ThreadCatcher();
Connection session = Jsoup.newSession();
Thread[] threads = new Thread[numThreads];
for (int threadNum = 0; threadNum < numThreads; threadNum++) {
Thread thread = new Thread(() -> {
for (int loop = 0; loop < numThreadLoops; loop++) {
for (int i = 0; i < urls.length; i++) {
try {
Document doc = session.newRequest().url(urls[i]).get();
assertEquals(titles[i], doc.title());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
});
thread.setName("Runner-" + threadNum);
thread.start();
thread.setUncaughtExceptionHandler(catcher);
threads[threadNum] = thread;
}
// now join them all
for (Thread thread : threads) {
thread.join();
}
assertEquals(0, catcher.exceptionCount.get());
}
// test that we throw a nice clear exception if you try to multi-thread by forget .newRequest()
@Test
public void multiThreadWithoutNewRequestBlowsUp() throws InterruptedException {
int numThreads = 5;
String url = SlowRider.Url + "?" + SlowRider.MaxTimeParam + "=20000"; // this makes sure that the first req is still executing whilst the others run
String title = "Slow Rider";
ThreadCatcher catcher = new ThreadCatcher();
Connection session = Jsoup.newSession();
// run first slow request
AtomicInteger successful = new AtomicInteger();
Thread slow = new Thread(() -> {
try {
Document doc = session.url(url).get();
assertNotNull(doc);
} catch (IOException e) {
if (!isInterruptedException(e))
throw new UncheckedIOException(e);
}
});
slow.start();
Thread.sleep(100); // yield so that thread can start before the next do
// spawn others, should fail
Thread[] threads = new Thread[numThreads];
for (int threadNum = 0; threadNum < numThreads; threadNum++) {
Thread thread = new Thread(() -> {
try {
Document doc = session.url(url).get();
assertEquals(title, doc.title());
successful.getAndIncrement();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
thread.setName("Runner-" + threadNum);
thread.setUncaughtExceptionHandler(catcher);
thread.start();
threads[threadNum] = thread;
}
// now join them all
for (Thread thread : threads) {
thread.join();
}
// cancel the slow runner so we can wrap up the test quicker
slow.interrupt();
// only one should have passed, rest should have blown up (assuming the started whilst other was running)
//assertEquals(numThreads - 1, catcher.multiThreadExceptions.get());
//assertEquals(numThreads - 1, catcher.exceptionCount.get());
/* The checks above work when all 20 threads are executed within 10 seconds. However, depending on how cloudy it
is when the CI jobs are run, they may not all complete in time. As of writing that appears most commonly on the
macOS runners, which appear overtaxed. That makes this test flaky. So we relax the test conditions, and make
sure at least just one passed and one failed. That's OK in prod as well, because we are only concerned about
concurrent execution, which the impl does detect correctly. */
assertEquals(0, successful.get());
assertTrue(catcher.multiThreadExceptions.get() > 0);
assertEquals(catcher.multiThreadExceptions.get(), catcher.exceptionCount.get()); // all exceptions are multi-threaded
}
@Test
public void multiThreadWithProgressListener() throws InterruptedException {
// tests that we can use one progress listener for multiple URLs and threads.
int numThreads = 10;
String[] urls = {
FileServlet.urlTo("/htmltests/medium.html"),
FileServlet.urlTo("/htmltests/upload-form.html"),
FileServlet.urlTo("/htmltests/comments.html"),
FileServlet.urlTo("/htmltests/large.html"),
};
Set<String> seenUrls = ConcurrentHashMap.newKeySet();
AtomicInteger completedCount = new AtomicInteger(0);
ThreadCatcher catcher = new ThreadCatcher();
Connection session = Jsoup.newSession()
.onResponseProgress((processed, total, percent, response) -> {
if (percent == 100.0f) {
//System.out.println("Completed " + Thread.currentThread().getName() + "- " + response.url());
seenUrls.add(response.url().toExternalForm());
completedCount.incrementAndGet();
}
});
Thread[] threads = new Thread[numThreads];
for (int threadNum = 0; threadNum < numThreads; threadNum++) {
Thread thread = new Thread(() -> {
for (String url : urls) {
try {
Connection con = session.newRequest().url(url);
con.get();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
});
thread.setName("Runner-" + threadNum);
thread.setUncaughtExceptionHandler(catcher);
thread.start();
threads[threadNum] = thread;
}
// now join them all
for (Thread thread : threads) {
thread.join();
}
assertEquals(0, catcher.exceptionCount.get());
assertEquals(urls.length, seenUrls.size());
assertEquals(urls.length * numThreads, completedCount.get());
}
static class ThreadCatcher implements Thread.UncaughtExceptionHandler {
AtomicInteger exceptionCount = new AtomicInteger();
AtomicInteger multiThreadExceptions = new AtomicInteger();
@Override
public void uncaughtException(Thread t, Throwable e) {
if (e.getMessage() != null && e.getMessage().contains("Multiple threads")) {
multiThreadExceptions.incrementAndGet();
} else if (!isInterruptedException(e)) {
e.printStackTrace();
}
exceptionCount.incrementAndGet();
}
}
private static boolean isInterruptedException(Throwable e) {
Throwable cause = e;
while (cause != null) {
if (cause instanceof InterruptedException) return true;
cause = cause.getCause();
}
return false;
}
}
================================================
FILE: src/test/java/org/jsoup/integration/SessionTest.java
================================================
package org.jsoup.integration;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.integration.servlets.CookieServlet;
import org.jsoup.integration.servlets.EchoServlet;
import org.jsoup.integration.servlets.FileServlet;
import org.jsoup.nodes.Document;
import org.jsoup.parser.Parser;
import org.jsoup.parser.Tag;
import org.jsoup.parser.TagSet;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class SessionTest {
@BeforeAll
public static void setUp() {
TestServer.start();
}
private static Elements keyEls(String key, Document doc) {
return doc.select("th:contains(" + key + ") + td");
}
private static String keyText(String key, Document doc) {
return doc.selectFirst("th:contains(" + key + ") + td").text();
}
@Test
public void testPathScopedCookies() throws IOException {
final Connection session = Jsoup.newSession();
final String userAgent = "Jsoup Testalot v0.1";
session.userAgent(userAgent);
session.url(CookieServlet.Url);
// should have no cookies:
Connection con1 = session.newRequest();
Document doc1 = con1.get();
assertEquals(0, doc1.select("table tr").size()); // none sent to servlet
// set the cookies
Connection con2 = session.newRequest().data(CookieServlet.SetCookiesParam, "1");
Document doc2 = con2.get();
assertEquals(0, doc2.select("table tr").size()); // none sent to servlet - we just got them!
Map<String, String> cookies = con2.response().cookies(); // simple cookie response, all named "One", so should be last sent
assertEquals(2, cookies.size());
assertEquals("EchoServlet", cookies.get("One"));
// test that all response cookies are present, even if would not be sent for this request path (i.e. cookies() res can be set on a req, without using sessions)
assertEquals("Override", cookies.get("Two"));
// todo - interrogate cookie-store
// check that they are sent and filtered to the right path
Connection con3 = session.newRequest();
Document doc3 = con3.get();
assertCookieServlet(doc3);
Document echo = session.newRequest().url(EchoServlet.Url).get();
assertEchoServlet(echo);
assertEquals(userAgent, keyText("User-Agent", echo)); // check that customer user agent sent on session arrived
// check that cookies aren't set out of the session
Document doc4 = Jsoup.newSession().url(CookieServlet.Url).get();
assertEquals(0, doc4.select("table tr").size()); // none sent to servlet
// check can add local ones also
Document doc5 = session.newRequest().cookie("Bar", "Qux").get();
Elements doc5Bar = keyEls("Bar", doc5);
assertEquals("Qux", doc5Bar.first().text());
}
// validate that only cookies set by cookie servlet get to the cookie servlet path
private void assertCookieServlet(Document doc) {
assertEquals(2, doc.select("table tr").size()); // two of three sent to servlet (/ and /CookieServlet)
Elements doc3Els = keyEls("One", doc);
assertEquals(2, doc3Els.size());
assertEquals("CookieServlet", doc3Els.get(0).text()); // ordered by most specific path
assertEquals("Root", doc3Els.get(1).text()); // ordered by most specific path
}
// validate that only for echo servlet
private void assertEchoServlet(Document doc) {
Elements echoEls = keyEls("Cookie: One", doc); // two of three sent to servlet (/ and /EchoServlet)
assertEquals(2, echoEls.size());
assertEquals("EchoServlet", echoEls.get(0).text()); // ordered by most specific path - /Echo
assertEquals("Root", echoEls.get(1).text()); // ordered by most specific path - /
}
@Test
public void testPathScopedCookiesOnRedirect() throws IOException {
Connection session = Jsoup.newSession();
Document doc1 = session.newRequest()
.url(CookieServlet.Url)
.data(CookieServlet.LocationParam, EchoServlet.Url)
.data(CookieServlet.SetCookiesParam, "1")
.get();
// we should be redirected to the echo servlet with cookies
assertEquals(EchoServlet.Url, doc1.location());
assertEchoServlet(doc1); // checks we only have /echo cookies
Document doc2 = session.newRequest()
.url(EchoServlet.Url)
.get();
assertEchoServlet(doc2); // test retained in session
Document doc3 = session.newRequest()
.url(CookieServlet.Url)
.get();
assertCookieServlet(doc3); // and so were the /cookie cookies
}
@Test
public void testCanChangeParsers() throws IOException {
Connection session = Jsoup.newSession().parser(Parser.xmlParser());
String xmlUrl = FileServlet.urlTo("/htmltests/xml-test.xml");
String xmlVal = "<doc><val>One<val>Two</val>Three</val></doc>\n";
Document doc1 = session.newRequest().url(xmlUrl).get();
assertEquals(xmlVal, doc1.html()); // not HTML normed, used XML parser
Document doc2 = session.newRequest().parser(Parser.htmlParser()).url(xmlUrl).get();
assertTrue(doc2.html().startsWith("<html>"));
Document doc3 = session.newRequest().url(xmlUrl).get();
assertEquals(xmlVal, doc3.html()); // did not blow away xml default
}
@Test
public void sessionTagSetDoesNotMutateRoot() {
Connection session = Jsoup.newSession();
TagSet rootTags = session.request().parser().tagSet();
int rootNamespacesBefore = tagSetNamespaceCount(rootTags);
Connection request = session.newRequest();
Parser parser = request.request().parser();
parser.parseInput("<custom>One <b>Two</b></custom>", "http://example.com/");
int rootNamespacesAfter = tagSetNamespaceCount(rootTags);
assertEquals(rootNamespacesBefore, rootNamespacesAfter);
}
@Test
public void sessionTagSetCustomizerDoesNotMutateRoot() {
Connection session = Jsoup.newSession();
TagSet rootTags = session.request().parser().tagSet();
rootTags.onNewTag(tag -> {
if (!tag.isKnownTag())
tag.set(Tag.RcData);
});
int rootNamespacesBefore = tagSetNamespaceCount(rootTags);
Connection request = session.newRequest();
Parser parser = request.request().parser();
Document doc = parser.parseInput("<custom>One <b>Two</b></custom>", "https://example.com/");
assertEquals(0, doc.select("custom b").size());
int rootNamespacesAfter = tagSetNamespaceCount(rootTags);
assertEquals(rootNamespacesBefore, rootNamespacesAfter);
}
private static int tagSetNamespaceCount(TagSet tagSet) {
try {
Field tagsField = TagSet.class.getDeclaredField("tags");
tagsField.setAccessible(true);
Map<?, ?> tags = (Map<?, ?>) tagsField.get(tagSet);
return tags.size();
} catch (ReflectiveOperationException e) {
throw new RuntimeException(e);
}
}
}
================================================
FILE: src/test/java/org/jsoup/integration/TestServer.java
================================================
package org.jsoup.integration;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.FilterMapping;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.StdErrLog;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.jsoup.integration.servlets.AuthFilter;
import org.jsoup.integration.servlets.BaseServlet;
import org.jsoup.integration.servlets.ProxyServlet;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.concurrent.atomic.AtomicInteger;
public class TestServer {
static int Port;
static int TlsPort;
private static final String Localhost = "localhost";
private static final String KeystorePassword = "hunter2";
private static final Server Jetty = newServer();
private static final ServletHandler JettyHandler = new ServletHandler();
private static final Server Proxy = newServer();
private static final Server AuthedProxy = newServer();
private static final HandlerWrapper ProxyHandler = new HandlerWrapper();
private static final HandlerWrapper AuthedProxyHandler = new HandlerWrapper();
private static final ProxySettings ProxySettings = new ProxySettings();
private static Server newServer() {
// logs to stdout, so not highlighted as errors in Maven test runs
StdErrLog logger = new StdErrLog();
logger.setStdErrStream(System.out);
Log.setLog(logger);
return new Server(new InetSocketAddress(Localhost, 0));
}
static {
Jetty.setHandler(JettyHandler);
Proxy.setHandler(ProxyHandler);
AuthedProxy.setHandler(AuthedProxyHandler);
// TLS setup:
try {
File keystoreFile = ParseTest.getFile("/local-cert/server.pfx");
if (!keystoreFile.exists()) throw new FileNotFoundException(keystoreFile.toString());
addHttpsConnector(keystoreFile, Jetty);
setupDefaultTrust(keystoreFile);
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
private TestServer() {
}
public static void start() {
synchronized (Jetty) {
if (Jetty.isStarted()) return;
try {
Jetty.start();
JettyHandler.addFilterWithMapping(new FilterHolder(new AuthFilter(false, false)), "/*", FilterMapping.ALL);
Connector[] jcons = Jetty.getConnectors();
Port = ((ServerConnector) jcons[0]).getLocalPort();
TlsPort = ((ServerConnector) jcons[1]).getLocalPort();
ProxyHandler.setHandler(ProxyServlet.createHandler(false)); // includes proxy, CONNECT proxy, and Auth filters
Proxy.start();
ProxySettings.port = ((ServerConnector) Proxy.getConnectors()[0]).getLocalPort();
AuthedProxyHandler.setHandler(ProxyServlet.createHandler(true));
AuthedProxy.start();
ProxySettings.authedPort = ((ServerConnector) AuthedProxy.getConnectors()[0]).getLocalPort();
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
}
/**
Close any current connections to the authed proxy. Tunneled connections only authenticate in their first
CONNECT, and may be kept alive and reused. So when we want to test unauthed - authed flows, we need to disconnect
them first.
*/
static int closeAuthedProxyConnections() {
ServerConnector connector = (ServerConnector) AuthedProxy.getConnectors()[0];
AtomicInteger count = new AtomicInteger();
connector.getConnectedEndPoints().forEach(endPoint -> {
endPoint.close();
count.getAndIncrement();
});
return count.get();
}
public static ServletUrls map(Class<? extends BaseServlet> servletClass) {
synchronized (Jetty) {
if (!Jetty.isStarted())
start(); // if running out of the test cases
String path = "/" + servletClass.getSimpleName();
JettyHandler.addServletWithMapping(servletClass, path + "/*");
String url = "http://" + Localhost + ":" + Port + path;
String tlsUrl = "https://" + Localhost + ":" + TlsPort + path;
return new ServletUrls(url, tlsUrl);
}
}
public static class ServletUrls {
public final String url;
public final String tlsUrl;
public ServletUrls(String url, String tlsUrl) {
this.url = url;
this.tlsUrl = tlsUrl;
}
}
public static ProxySettings proxySettings() {
synchronized (Jetty) {
if (!Jetty.isStarted())
start();
return ProxySettings;
}
}
//public static String proxy
public static class ProxySettings {
final String hostname = Localhost;
int port;
int authedPort;
}
private static void addHttpsConnector(File keystoreFile, Server server) {
// Cribbed from https://github.com/jetty/jetty.project/blob/jetty-9.4.x/examples/embedded/src/main/java/org/eclipse/jetty/embedded/LikeJettyXml.java
SslContextFactory sslContextFactory = new SslContextFactory.Server();
String path = keystoreFile.getAbsolutePath();
sslContextFactory.setKeyStorePath(path);
sslContextFactory.setKeyStorePassword(KeystorePassword);
sslContextFactory.setKeyManagerPassword(KeystorePassword);
sslContextFactory.setTrustStorePath(path);
sslContextFactory.setTrustStorePassword(KeystorePassword);
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
httpsConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector sslConnector = new ServerConnector(
server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(httpsConfig));
sslConnector.setHost(Localhost);
server.addConnector(sslConnector);
}
private static void setupDefaultTrust(File keystoreFile) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException {
// Configure HttpsUrlConnection (jsoup) to trust (only) this cert
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(Files.newInputStream(keystoreFile.toPath()), KeystorePassword.toCharArray());
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(trustStore);
TrustManager[] managers = trustManagerFactory.getTrustManagers();
SSLContext tls = SSLContext.getInstance("TLS");
tls.init(null, managers, null);
SSLContext.setDefault(tls);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/AuthFilter.java
================================================
package org.jsoup.integration.servlets;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
/**
A filter to test basic authenticated requests. If the request header "X-Wants-Authentication" is set, or if
alwaysWantsAuth is enabled, the filter is invoked, and requests must send the correct user authentication details.
*/
public class AuthFilter implements Filter {
public static final String WantsServerAuthentication = "X-Wants-ServerAuthentication";
public static final String ServerUser = "admin";
public static final String ServerRealm = "jsoup test server authentication realm";
private static volatile String ServerPassword = newServerPassword();
public static final String WantsProxyAuthentication = "X-Wants-ProxyAuthentication";
public static final String ProxyUser = "foxyproxy";
public static final String ProxyRealm = "jsoup test proxy authentication realm";
private static volatile String ProxyPassword = newProxyPassword();
private final boolean alwaysWantsAuth; // we run a particular port that always wants auth - so the CONNECT tunnels can be authed. (The Java proxy tunnel CONNECT request strips the wants-auth headers)
private final boolean forProxy;
private final String wantsHeader;
private final String authorizationHeader;
/**
Creates an Authentication Filter with hardcoded credential expectations.
* @param alwaysWantsAuth true if this filter should always check for authentication, regardless of the Wants Auth header
* @param forProxy true if this wraps a Proxy and should use Proxy-Authenticate headers, credentials etc. False
* if wrapping the web server.
*/
public AuthFilter(boolean alwaysWantsAuth, boolean forProxy) {
this.alwaysWantsAuth = alwaysWantsAuth;
this.forProxy = forProxy;
wantsHeader = forProxy ? WantsProxyAuthentication : WantsServerAuthentication;
authorizationHeader = forProxy ? "Proxy-Authorization" : "Authorization";
}
private static String newPassword() {
return "pass-" + Math.random();
}
// passwords get rotated in tests so that Java's auth cache is invalidated and a new auth callback occurs.
// requires tests hitting these are called serially.
public static String newServerPassword() {
return ServerPassword = newPassword() + "-server";
}
public static String newProxyPassword() {
return ProxyPassword = newPassword() + "-proxy";
}
@Override public void init(FilterConfig filterConfig) throws ServletException {}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
HttpServletRequest req = (HttpServletRequest) request;
HttpServletResponse res = (HttpServletResponse) response;
boolean accessGranted = checkAuth(req);
if (accessGranted) {
chain.doFilter(request, response);
return;
}
// Wants but failed auth - send appropriate header:
if (forProxy) {
res.setHeader("Proxy-Authenticate", "Basic realm=\"" + ProxyRealm + "\"");
// ^^ Duped in ProxyServlet for CONNECT
res.sendError(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED);
} else {
res.setHeader("WWW-Authenticate", "Basic realm=\"" + ServerRealm + "\"");
res.sendError(HttpServletResponse.SC_UNAUTHORIZED);
}
}
@Override public void destroy() {}
public boolean checkAuth(HttpServletRequest req) {
if (alwaysWantsAuth || req.getHeader(wantsHeader) != null) {
String authHeader = req.getHeader(authorizationHeader);
if (authHeader != null) {
int space = authHeader.indexOf(' ');
if (space > 0) {
String value = authHeader.substring(space + 1);
String expected = forProxy ?
(ProxyUser + ":" + ProxyPassword) :
(ServerUser + ":" + ServerPassword);
String base64 = Base64.getEncoder().encodeToString(expected.getBytes(StandardCharsets.UTF_8));
return base64.equals(value); // if passed auth
}
}
return false; // unexpected header value
}
return true; // auth not required
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/BaseServlet.java
================================================
package org.jsoup.integration.servlets;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public abstract class BaseServlet extends HttpServlet {
static final String TextHtml = "text/html; charset=UTF-8";
abstract protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException;
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
doIt(req, res);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
doIt(req, res);
}
@Override
protected void doPut(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
doIt(req, res);
}
@Override
protected void doDelete(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
doIt(req, res);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/CookieServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
public class CookieServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(CookieServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
public static final String SetCookiesParam = "setCookies";
public static final String LocationParam = "loc";
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
// Do we want to set cookies?
if (req.getParameter(SetCookiesParam) != null)
setCookies(res);
// Do we want to redirect elsewhere?
String loc = req.getParameter(LocationParam);
if (loc != null) {
res.sendRedirect(loc);
return;
}
// print out the cookies that were received
res.setContentType(TextHtml);
res.setStatus(200);
PrintWriter w = res.getWriter();
w.println("<table>");
final Cookie[] cookies = req.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
EchoServlet.write(w, cookie.getName(), cookie.getValue());
}
}
w.println("</table>");
}
private void setCookies(HttpServletResponse res) {
Cookie one = new Cookie("One", "Root");
one.setPath("/");
res.addCookie(one);
Cookie two = new Cookie("One", "CookieServlet");
two.setPath("/CookieServlet");
two.setHttpOnly(true);
two.setComment("Quite nice");
res.addCookie(two);
Cookie three = new Cookie("One", "EchoServlet");
three.setPath("/EchoServlet");
res.addCookie(three);
Cookie four = new Cookie("Two", "NoSuchPath");
four.setPath("/bogus");
res.addCookie(four);
Cookie five = new Cookie("Two", "Override");
five.setPath("/bogus");
res.addCookie(five);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/DeflateServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
public class DeflateServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(DeflateServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
res.setContentType(TextHtml);
res.setStatus(HttpServletResponse.SC_OK);
res.setHeader("Content-Encoding", "deflate");
String doc = "<p>Hello, World!<p>That should be enough, right?<p>Hello, World!<p>That should be enough, right?";
DeflaterOutputStream stream = new DeflaterOutputStream(
res.getOutputStream(),
new Deflater(Deflater.BEST_COMPRESSION, true)); // true = nowrap zlib headers
stream.write(doc.getBytes(StandardCharsets.UTF_8));
stream.close();
}
// allow the servlet to run as a main program, for local test
public static void main(String[] args) {
TestServer.start();
System.out.println(Url);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/EchoServlet.java
================================================
package org.jsoup.integration.servlets;
import org.eclipse.jetty.server.Request;
import org.jsoup.helper.DataUtil;
import org.jsoup.internal.StringUtil;
import org.jsoup.integration.TestServer;
import javax.servlet.MultipartConfigElement;
import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.Part;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Enumeration;
import static org.jsoup.nodes.Entities.escape;
public class EchoServlet extends BaseServlet {
public static final String CodeParam = "code";
private static final int DefaultCode = HttpServletResponse.SC_OK;
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(EchoServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
int intCode = DefaultCode;
String code = req.getHeader(CodeParam);
if (code != null)
intCode = Integer.parseInt(code);
boolean isMulti = maybeEnableMultipart(req);
res.setContentType(TextHtml);
res.setStatus(intCode);
// no-cache headers for test
res.addHeader("Cache-Control", "no-cache");
res.addHeader("Cache-Control", "no-store");
PrintWriter w = res.getWriter();
w.write("<title>Webserver Environment Variables\n" +
" \n" +
" \n" +
" ");
// some get items
write(w, "Method", req.getMethod());
write(w, "Request URI", req.getRequestURI());
write(w, "Path Info", req.getPathInfo());
write(w, "Query String", req.getQueryString());
// request headers (why is it an enumeration?)
Enumeration headerNames = req.getHeaderNames();
while (headerNames.hasMoreElements()) {
String header = headerNames.nextElement();
Enumeration headers = req.getHeaders(header);
while (headers.hasMoreElements()) {
write(w, header, headers.nextElement());
}
}
// cookies
final Cookie[] cookies = req.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
EchoServlet.write(w, "Cookie: " + cookie.getName(), cookie.getValue());
}
}
// the request params
Enumeration parameterNames = req.getParameterNames();
while (parameterNames.hasMoreElements()) {
String name = parameterNames.nextElement();
String[] values = req.getParameterValues(name);
write(w, name, StringUtil.join(values, ", "));
}
// post body
ByteBuffer byteBuffer = DataUtil.readToByteBuffer(req.getInputStream(), 0);
String postData = new String(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit(), StandardCharsets.UTF_8);
if (!StringUtil.isBlank(postData)) {
write(w, "Post Data", postData);
}
// file uploads
if (isMulti) {
Collection parts = req.getParts();
write(w, "Parts", String.valueOf(parts.size()));
for (Part part : parts) {
String name = part.getName();
write(w, "Part " + name + " ContentType", part.getContentType());
write(w, "Part " + name + " Name", name);
write(w, "Part " + name + " Filename", part.getSubmittedFileName());
write(w, "Part " + name + " Size", String.valueOf(part.getSize()));
part.delete();
}
}
w.println("
");
}
static void write(PrintWriter w, String key, String val) {
w.println("| " + escape(key) + " | " + escape(val) + " |
|---|
");
}
// allow the servlet to run as a main program, for local test
public static void main(String[] args) {
TestServer.start();
System.out.println("Listening on " + Url + " and " + TlsUrl);
}
private static boolean maybeEnableMultipart(HttpServletRequest req) {
boolean isMulti = req.getContentType() != null
&& req.getContentType().startsWith("multipart/form-data");
if (isMulti) {
req.setAttribute(Request.MULTIPART_CONFIG_ELEMENT, new MultipartConfigElement(
System.getProperty("java.io.tmpdir")));
}
return isMulti;
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/FileServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.ParseTest;
import org.jsoup.integration.TestServer;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
public class FileServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(FileServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
public static final String ContentTypeParam = "contentType";
public static final String HtmlType = "text/html";
static final String XmlType = "text/xml";
public static final String SuppressContentLength = "surpriseMe";
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
String contentType = req.getParameter(ContentTypeParam);
if (contentType == null) {
contentType = HtmlType;
if (req.getPathInfo().contains(".xml")) contentType = XmlType;
}
String location = req.getPathInfo();
File file = ParseTest.getFile(location);
if (file.exists()) {
res.setContentType(contentType);
if (file.getName().endsWith("gz"))
res.addHeader("Content-Encoding", "gzip");
if (req.getParameter(SuppressContentLength) == null)
res.setContentLength((int) file.length());
res.setStatus(HttpServletResponse.SC_OK);
ServletOutputStream out = res.getOutputStream();
Files.copy(file.toPath(), out);
out.flush();
} else {
res.sendError(HttpServletResponse.SC_NOT_FOUND);
}
}
public static String urlTo(String path) {
return Url + path;
}
public static String tlsUrlTo(String path) {
return TlsUrl + path;
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/HelloServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public class HelloServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(HelloServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
res.setContentType(TextHtml);
res.setStatus(HttpServletResponse.SC_OK);
String doc = "Hello, World!";
res.getWriter().write(doc);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/InterruptedServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import org.jsoup.parser.CharacterReaderTest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public class InterruptedServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(InterruptedServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
public static final String Magnitude = "magnitude";
public static final String Larger = "larger";
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
String magnitude = req.getParameter(Magnitude);
magnitude = magnitude == null ? "" : magnitude;
res.setContentType(TextHtml);
res.setStatus(HttpServletResponse.SC_OK);
StringBuilder sb = new StringBuilder();
sb.append("
Something");
while (sb.length() <= 32 * 1024) {
sb.append("A suitable amount of data.
\n");
}
sb.append("Finale.
");
String data = sb.toString();
int contentLength = magnitude.equals(Larger) ? data.length() * 2 : data.length() / 2;
res.setContentLength(contentLength);
res.getWriter().write(data);
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/ProxyServlet.java
================================================
package org.jsoup.integration.servlets;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.proxy.AsyncProxyServlet;
import org.eclipse.jetty.proxy.ConnectHandler;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.FilterMapping;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.jsoup.integration.TestServer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import static org.jsoup.integration.servlets.AuthFilter.ProxyRealm;
public class ProxyServlet extends AsyncProxyServlet {
public static TestServer.ProxySettings ProxySettings = TestServer.proxySettings();
public static String Via = "1.1 jsoup test proxy";
static {
System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
// removes Basic, which is otherwise excluded from auth for CONNECT tunnels
}
public static Handler createHandler(boolean alwaysAuth) {
// ConnectHandler wraps this ProxyServlet and handles CONNECT, which sets up a tunnel for HTTPS requests and is
// opaque to the proxy. The ProxyServlet handles simple HTTP requests.
AuthFilter authFilter = new AuthFilter(alwaysAuth, true);
ConnectHandler connectHandler = new ConnectProxy(authFilter);
ServletHandler proxyHandler = new ServletHandler();
proxyHandler.addFilterWithMapping(new FilterHolder(authFilter), "/*", FilterMapping.ALL); // auth for HTTP proxy
ServletHolder proxyServletHolder = new ServletHolder(ProxyServlet.class); // Holder wraps as it requires maxThreads initialization
proxyServletHolder.setAsyncSupported(true);
proxyServletHolder.setInitParameter("maxThreads", "200");
proxyHandler.addServletWithMapping(proxyServletHolder, "/*");
connectHandler.setHandler(proxyHandler);
return connectHandler;
}
@Override
protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) {
super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse);
proxyResponse.addHeader("Via", Via);
}
/** Supports CONNECT tunnels */
static class ConnectProxy extends ConnectHandler {
final AuthFilter authFilter;
public ConnectProxy(AuthFilter authFilter) {
this.authFilter = authFilter;
}
@Override
protected boolean handleAuthentication(HttpServletRequest req, HttpServletResponse res, String address) {
boolean accessGranted = authFilter.checkAuth(req);
//System.err.println("CONNECT AUTH: " + accessGranted);
// need to add the desired auth header if not granted. Returning false here will also send 407 header
if (!accessGranted) {
res.setHeader("Proxy-Authenticate", "Basic realm=\"" + ProxyRealm + "\"");
}
return accessGranted;
}
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/RedirectServlet.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public class RedirectServlet extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(RedirectServlet.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
public static final String LocationParam = "loc";
public static final String CodeParam = "code";
public static final String SetCookiesParam = "setCookies";
private static final int DefaultCode = HttpServletResponse.SC_MOVED_TEMPORARILY;
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
String location = req.getParameter(LocationParam);
if (location == null)
location = "";
int intCode = DefaultCode;
String code = req.getParameter(CodeParam);
if (code != null)
intCode = Integer.parseInt(code);
if (req.getParameter(SetCookiesParam) != null) {
res.addCookie(new Cookie("token", "asdfg123"));
res.addCookie(new Cookie("uid", "foobar"));
res.addCookie(new Cookie("uid", "jhy")); // dupe, should use latter
}
res.setHeader("Location", location);
res.setStatus(intCode);
res.flushBuffer();
}
}
================================================
FILE: src/test/java/org/jsoup/integration/servlets/SlowRider.java
================================================
package org.jsoup.integration.servlets;
import org.jsoup.integration.TestServer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
/**
* Slowly, interminably writes output. For the purposes of testing timeouts and interrupts.
*/
public class SlowRider extends BaseServlet {
public static final String Url;
public static final String TlsUrl;
static {
TestServer.ServletUrls urls = TestServer.map(SlowRider.class);
Url = urls.url;
TlsUrl = urls.tlsUrl;
}
private static final int SleepTime = 2000;
public static final String MaxTimeParam = "maxTime";
public static final String IntroSizeParam = "introSize";
@Override
protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {
pause(1000);
res.setContentType(TextHtml);
res.setStatus(HttpServletResponse.SC_OK);
PrintWriter w = res.getWriter();
int maxTime = -1;
String maxTimeP = req.getParameter(MaxTimeParam);
if (maxTimeP != null) {
maxTime = Integer.parseInt(maxTimeP);
}
int introSize = 0;
String introSizeP = req.getParameter(IntroSizeParam);
if (introSizeP != null) {
introSize = Integer.parseInt(introSizeP);
}
long startTime = System.currentTimeMillis();
w.println("Slow Rider");
// write out a bunch of stuff at the start before interim pauses, gets past some buffers
if (introSize != 0) {
StringBuilder s = new StringBuilder();
while (s.length() < introSize) {
s.append("Hello and welcome to the Slow Rider!
\n");
}
w.println(s);
w.flush();
}
while (true) {
w.println("Are you still there?");
boolean err = w.checkError(); // flush, and check still ok
if (err) {
log("Remote connection lost");
break;
}
if (pause(SleepTime)) break;
if (maxTime > 0 && System.currentTimeMillis() > startTime + maxTime) {
w.println("
outatime
");
break;
}
}
}
private static boolean pause(int sleepTime) {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
return true;
}
return false;
}
// allow the servlet to run as a main program, for local test
public static void main(String[] args) {
TestServer.start();
System.out.println(Url);
}
}
================================================
FILE: src/test/java/org/jsoup/internal/ControllableInputStreamTest.java
================================================
package org.jsoup.internal;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.*;
class ControllableInputStreamTest {
@Test
void respectsMaxCapDuringFill() throws IOException {
byte[] data = "0123456789".getBytes(); // 10 bytes
CountingInputStream counting = new CountingInputStream(new ByteArrayInputStream(data));
ControllableInputStream in = ControllableInputStream.wrap(counting, 5); // cap at 5 bytes
byte[] buf = new byte[10];
int read = in.read(buf);
assertEquals(5, read, "should only read up to cap");
assertEquals(5, counting.count, "underlying stream should not be pulled past cap");
assertFalse(in.baseReadFully(), "cap hit is not EOF");
int second = in.read(buf);
assertEquals(-1, second, "further reads return -1 once cap is exhausted");
assertFalse(in.baseReadFully(), "still not true EOF");
in.close();
}
@Test
void compactsBufferWithActiveMark() throws IOException {
int size = SharedConstants.DefaultBufferSize * 2;
byte[] data = new byte[size];
for (int i = 0; i < size; i++) data[i] = (byte) (i % 256);
ControllableInputStream in = ControllableInputStream.wrap(new ByteArrayInputStream(data), 0);
byte[] first = new byte[500];
assertEquals(500, in.read(first));
in.mark(SharedConstants.DefaultBufferSize); // mark at logical pos 500
byte[] consume = new byte[SharedConstants.DefaultBufferSize];
int firstRead = in.read(consume); // serves remainder of current buffer (BufferSize - 500)
assertEquals(SharedConstants.DefaultBufferSize - 500, firstRead);
byte[] more = new byte[1000];
int secondRead = in.read(more); // triggers fill() with active mark, then consumes from freshly filled buffer
assertEquals(SharedConstants.DefaultBufferSize - firstRead, secondRead);
in.reset(); // should rewind to mark despite prior compaction
byte[] reread = new byte[1000];
assertEquals(1000, in.read(reread));
for (int i = 0; i < reread.length; i++) {
assertEquals(data[500 + i], reread[i], "byte mismatch at " + i);
}
in.close();
}
private static final class CountingInputStream extends FilterInputStream {
int count = 0;
CountingInputStream(InputStream in) {
super(in);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
int r = super.read(b, off, len);
if (r > 0) count += r;
return r;
}
@Override
public int read() throws IOException {
int r = super.read();
if (r != -1) count++;
return r;
}
}
}
================================================
FILE: src/test/java/org/jsoup/internal/QuietAppendableTest.java
================================================
package org.jsoup.internal;
import org.jsoup.Jsoup;
import org.jsoup.SerializationException;
import org.jsoup.nodes.Document;
import org.junit.jupiter.api.Test;
import java.io.CharArrayWriter;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*;
class QuietAppendableTest {
@Test void wrap() {
assertInstanceOf(
QuietAppendable.StringBuilderAppendable.class,
QuietAppendable.wrap(new StringBuilder())
);
assertInstanceOf(
QuietAppendable.BaseAppendable.class,
QuietAppendable.wrap(new CharArrayWriter())
);
}
@Test void supplemental() {
// hits append(char[] chars, int offset, int len) with len 2 for supplemental codepoint
String expect = "😀";
char[] chars = new char[2];
chars[0] = expect.charAt(0);
chars[1] = expect.charAt(1);
assertEquals(2, expect.length());
QuietAppendable sb = QuietAppendable.wrap(new StringBuilder());
sb.append(chars, 0, 2);
String s = sb.toString();
assertEquals(expect, s);
CharArrayWriter cw = new CharArrayWriter();
QuietAppendable qa = QuietAppendable.wrap(cw);
qa.append(chars, 0, 2);
String out = cw.toString();
assertEquals(expect, out);
}
private static Appendable brokenAppender() {
// returns an Appendable that throws an IOException on any put
return new Appendable() {
@Override
public Appendable append(CharSequence csq) throws IOException {
throw new IOException("broken");
}
@Override
public Appendable append(CharSequence csq, int start, int end) throws IOException {
throw new IOException("broken");
}
@Override
public Appendable append(char c) throws IOException {
throw new IOException("broken");
}
};
}
@Test void appendThrowsSerializationException() {
Document doc = Jsoup.parse("");
Appendable brokenWriter = brokenAppender();
boolean threw = false;
try {
doc.html(brokenWriter);
} catch (SerializationException e) {
threw = true;
Throwable cause = e.getCause();
assertEquals("broken", cause.getMessage());
assertInstanceOf(IOException.class, cause);
}
assertTrue(threw);
}
}
================================================
FILE: src/test/java/org/jsoup/internal/ReaderTest.java
================================================
package org.jsoup.internal;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.parser.CharacterReader;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Test;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.jsoup.integration.ParseTest.getPath;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class ReaderTest {
@Test void readerOfStringAndFile() throws IOException {
// make sure that reading from a String and from a File produce the same bytes
Path path = getPath("/fuzztests/garble.html");
byte[] bytes = Files.readAllBytes(path);
String fromBytes = new String(bytes, StandardCharsets.UTF_8);
SimpleStreamReader streamReader = getReader(path);
String fromStream = getString(streamReader);
assertEquals(fromBytes, fromStream);
SimpleStreamReader reader2 = getReader(path);
CharacterReader cr = new CharacterReader(reader2);
String fullRead = cr.consumeTo('X'); // does not exist in input
assertEquals(fromBytes, fullRead);
}
private static String getString(SimpleStreamReader streamReader) throws IOException {
// read streamreader to a string:
StringBuilder builder = new StringBuilder();
char[] cbuffer = new char[1024];
int read;
while ((read = streamReader.read(cbuffer)) != -1) {
builder.append(cbuffer, 0, read);
}
return builder.toString();
}
private static SimpleStreamReader getReader(Path path) throws IOException {
// set up a chain as in when we parse: simplebufferedinput -> controllableinputstream -> simplestreamreader -> characterreader
SimpleBufferedInput input = new SimpleBufferedInput(Files.newInputStream(path));
ControllableInputStream stream = ControllableInputStream.wrap(input, 0);
return new SimpleStreamReader(stream, StandardCharsets.UTF_8);
}
}
================================================
FILE: src/test/java/org/jsoup/internal/SoftPoolTest.java
================================================
package org.jsoup.internal;
import org.junit.jupiter.api.Test;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.*;
public class SoftPoolTest {
private static final int BufSize = 12;
private static final int NumThreads = 5;
private static final int NumObjects = 3;
@Test
public void testSoftLocalPool() throws InterruptedException {
SoftPool
softLocalPool = new SoftPool<>(() -> new char[BufSize]);
ExecutorService executorService = Executors.newFixedThreadPool(NumThreads);
CountDownLatch latch = new CountDownLatch(NumThreads);
Set allBuffers = new HashSet<>();
Set[] threadLocalBuffers = new Set[NumThreads];
for (int i = 0; i < NumThreads; i++) {
threadLocalBuffers[i] = new HashSet<>();
}
AtomicInteger threadCount = new AtomicInteger();
Runnable task = () -> {
try {
int threadIndex = threadCount.getAndIncrement();
Set localBuffers = new HashSet<>();
// First borrow
for (int i = 0; i < NumObjects; i++) {
char[] buffer = softLocalPool.borrow();
assertEquals(BufSize, buffer.length);
localBuffers.add(buffer);
}
// Release buffers back to the pool
for (char[] buffer : localBuffers) {
softLocalPool.release(buffer);
}
// Borrow again and ensure buffers are reused
for (int i = 0; i < NumObjects; i++) {
char[] buffer = softLocalPool.borrow();
assertTrue(localBuffers.contains(buffer), "Buffer was not reused in the same thread");
threadLocalBuffers[threadIndex].add(buffer);
}
synchronized (allBuffers) {
allBuffers.addAll(threadLocalBuffers[threadIndex]);
}
} finally {
latch.countDown();
}
};
// Run the tasks
for (int i = 0; i < NumThreads; i++) {
executorService.submit(task::run);
}
// Wait for all threads to complete
latch.await();
executorService.shutdown();
// Ensure no buffers are shared between threads
Set uniqueBuffers = new HashSet<>();
for (Set bufferSet : threadLocalBuffers) {
for (char[] buffer : bufferSet) {
assertTrue(uniqueBuffers.add(buffer), "Buffer was shared between threads");
}
}
}
@Test
public void testSoftReferenceBehavior() {
SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);
// Borrow and release an object
char[] buffer = softLocalPool.borrow();
assertEquals(BufSize, buffer.length);
softLocalPool.release(buffer);
// Fake a GC
softLocalPool.threadLocalStack.get().clear();
// Ensure the object is garbage collected
assertNull(softLocalPool.threadLocalStack.get().get());
char[] second = softLocalPool.borrow();
// should be different, but same size
assertNotEquals(buffer, second);
assertEquals(BufSize, second.length);
}
@Test
public void testBorrowFromEmptyPool() {
SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);
// Borrow from an empty pool
char[] buffer = softLocalPool.borrow();
assertNotNull(buffer, "Borrowed null from an empty pool");
assertEquals(BufSize, buffer.length);
}
@Test
public void testReleaseMoreThanMaxIdle() {
SoftPool softLocalPool = new SoftPool<>(() -> new char[BufSize]);
// Borrow more than MaxIdle objects
List borrowedBuffers = new ArrayList<>();
for (int i = 0; i < SoftPool.MaxIdle + 5; i++) {
char[] buffer = softLocalPool.borrow();
borrowedBuffers.add(buffer);
}
// Release all borrowed objects back to the pool
for (char[] buffer : borrowedBuffers) {
softLocalPool.release(buffer);
}
// Ensure the pool size does not exceed MaxIdle
ArrayDeque stack = softLocalPool.getStack();
assertTrue(stack.size() <= SoftPool.MaxIdle, "Pool size exceeded MaxIdle limit");
}
}
================================================
FILE: src/test/java/org/jsoup/internal/StringUtilTest.java
================================================
package org.jsoup.internal;
import org.jsoup.Jsoup;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import static org.jsoup.internal.StringUtil.normaliseWhitespace;
import static org.jsoup.internal.StringUtil.resolve;
import static org.junit.jupiter.api.Assertions.*;
public class StringUtilTest {
@Test
public void join() {
assertEquals("", StringUtil.join(Collections.singletonList(""), " "));
assertEquals("one", StringUtil.join(Collections.singletonList("one"), " "));
assertEquals("one two three", StringUtil.join(Arrays.asList("one", "two", "three"), " "));
}
@Test public void padding() {
assertEquals("", StringUtil.padding(0));
assertEquals(" ", StringUtil.padding(1));
assertEquals(" ", StringUtil.padding(2));
assertEquals(" ", StringUtil.padding(15));
assertEquals(" ", StringUtil.padding(45)); // we default to tap out at 30
// memoization is up to 21 blocks (0 to 20 spaces) and exits early before min checks making maxPaddingWidth unused
assertEquals("", StringUtil.padding(0, -1));
assertEquals(" ", StringUtil.padding(20, -1));
// this test escapes memoization and continues through
assertEquals(" ", StringUtil.padding(21, -1));
// this test escapes memoization and using unlimited length (-1) will allow requested spaces
assertEquals(" ", StringUtil.padding(30, -1));
assertEquals(" ", StringUtil.padding(45, -1));
// we tap out at 0 for this test
assertEquals("", StringUtil.padding(0, 0));
// as memoization is escaped, setting zero for max padding will not allow any requested width
assertEquals("", StringUtil.padding(21, 0));
// we tap out at 30 for these tests making > 30 use 30
assertEquals("", StringUtil.padding(0, 30));
assertEquals(" ", StringUtil.padding(1, 30));
assertEquals(" ", StringUtil.padding(2, 30));
assertEquals(" ", StringUtil.padding(15, 30));
assertEquals(" ", StringUtil.padding(45, 30));
// max applies regardless of memoized
assertEquals(5, StringUtil.padding(20, 5).length());
}
@Test public void paddingInACan() {
String[] padding = StringUtil.padding;
assertEquals(21, padding.length);
for (int i = 0; i < padding.length; i++) {
assertEquals(i, padding[i].length());
}
}
@Test public void isBlank() {
assertTrue(StringUtil.isBlank(null));
assertTrue(StringUtil.isBlank(""));
assertTrue(StringUtil.isBlank(" "));
assertTrue(StringUtil.isBlank(" \r\n "));
assertFalse(StringUtil.isBlank("hello"));
assertFalse(StringUtil.isBlank(" hello "));
}
@Test public void isNumeric() {
assertFalse(StringUtil.isNumeric(null));
assertFalse(StringUtil.isNumeric(" "));
assertFalse(StringUtil.isNumeric("123 546"));
assertFalse(StringUtil.isNumeric("hello"));
assertFalse(StringUtil.isNumeric("123.334"));
assertTrue(StringUtil.isNumeric("1"));
assertTrue(StringUtil.isNumeric("1234"));
}
@Test public void isWhitespace() {
assertTrue(StringUtil.isWhitespace('\t'));
assertTrue(StringUtil.isWhitespace('\n'));
assertTrue(StringUtil.isWhitespace('\r'));
assertTrue(StringUtil.isWhitespace('\f'));
assertTrue(StringUtil.isWhitespace(' '));
assertFalse(StringUtil.isWhitespace('\u00a0'));
assertFalse(StringUtil.isWhitespace('\u2000'));
assertFalse(StringUtil.isWhitespace('\u3000'));
}
@Test public void normaliseWhiteSpace() {
assertEquals(" ", normaliseWhitespace(" \r \n \r\n"));
assertEquals(" hello there ", normaliseWhitespace(" hello \r \n there \n"));
assertEquals("hello", normaliseWhitespace("hello"));
assertEquals("hello there", normaliseWhitespace("hello\nthere"));
}
@Test public void normaliseWhiteSpaceHandlesHighSurrogates() {
String test71540chars = "\ud869\udeb2\u304b\u309a 1";
String test71540charsExpectedSingleWhitespace = "\ud869\udeb2\u304b\u309a 1";
assertEquals(test71540charsExpectedSingleWhitespace, normaliseWhitespace(test71540chars));
String extractedText = Jsoup.parse(test71540chars).text();
assertEquals(test71540charsExpectedSingleWhitespace, extractedText);
}
@Test public void resolvesRelativeUrls() {
assertEquals("http://example.com/one/two?three", resolve("http://example.com", "./one/two?three"));
assertEquals("http://example.com/one/two?three", resolve("http://example.com?one", "./one/two?three"));
assertEquals("http://example.com/one/two?three#four", resolve("http://example.com", "./one/two?three#four"));
assertEquals("https://example.com/one", resolve("http://example.com/", "https://example.com/one"));
assertEquals("http://example.com/one/two.html", resolve("http://example.com/two/", "../one/two.html"));
assertEquals("https://example2.com/one", resolve("https://example.com/", "//example2.com/one"));
assertEquals("https://example.com:8080/one", resolve("https://example.com:8080", "./one"));
assertEquals("https://example2.com/one", resolve("http://example.com/", "https://example2.com/one"));
assertEquals("https://example.com/one", resolve("wrong", "https://example.com/one"));
assertEquals("https://example.com/one", resolve("https://example.com/one", ""));
assertEquals("", resolve("wrong", "also wrong"));
assertEquals("ftp://example.com/one", resolve("ftp://example.com/two/", "../one"));
assertEquals("ftp://example.com/one/two.c", resolve("ftp://example.com/one/", "./two.c"));
assertEquals("ftp://example.com/one/two.c", resolve("ftp://example.com/one/", "two.c"));
// examples taken from rfc3986 section 5.4.2
assertEquals("http://example.com/g", resolve("http://example.com/b/c/d;p?q", "../../../g"));
assertEquals("http://example.com/g", resolve("http://example.com/b/c/d;p?q", "../../../../g"));
assertEquals("http://example.com/g", resolve("http://example.com/b/c/d;p?q", "/./g"));
assertEquals("http://example.com/g", resolve("http://example.com/b/c/d;p?q", "/../g"));
assertEquals("http://example.com/b/c/g.", resolve("http://example.com/b/c/d;p?q", "g."));
assertEquals("http://example.com/b/c/.g", resolve("http://example.com/b/c/d;p?q", ".g"));
assertEquals("http://example.com/b/c/g..", resolve("http://example.com/b/c/d;p?q", "g.."));
assertEquals("http://example.com/b/c/..g", resolve("http://example.com/b/c/d;p?q", "..g"));
assertEquals("http://example.com/b/g", resolve("http://example.com/b/c/d;p?q", "./../g"));
assertEquals("http://example.com/b/c/g/", resolve("http://example.com/b/c/d;p?q", "./g/."));
assertEquals("http://example.com/b/c/g/h", resolve("http://example.com/b/c/d;p?q", "g/./h"));
assertEquals("http://example.com/b/c/h", resolve("http://example.com/b/c/d;p?q", "g/../h"));
assertEquals("http://example.com/b/c/g;x=1/y", resolve("http://example.com/b/c/d;p?q", "g;x=1/./y"));
assertEquals("http://example.com/b/c/y", resolve("http://example.com/b/c/d;p?q", "g;x=1/../y"));
assertEquals("http://example.com/b/c/g?y/./x", resolve("http://example.com/b/c/d;p?q", "g?y/./x"));
assertEquals("http://example.com/b/c/g?y/../x", resolve("http://example.com/b/c/d;p?q", "g?y/../x"));
assertEquals("http://example.com/b/c/g#s/./x", resolve("http://example.com/b/c/d;p?q", "g#s/./x"));
assertEquals("http://example.com/b/c/g#s/../x", resolve("http://example.com/b/c/d;p?q", "g#s/../x"));
}
@Test void stripsControlCharsFromUrls() {
// should resovle to an absolute url:
assertEquals("foo:bar", resolve("\nhttps://\texample.com/", "\r\nfo\to:ba\br"));
}
@Test void allowsSpaceInUrl() {
assertEquals("https://example.com/foo bar/", resolve("HTTPS://example.com/example/", "../foo bar/"));
}
@Test
void isAscii() {
assertTrue(StringUtil.isAscii(""));
assertTrue(StringUtil.isAscii("example.com"));
assertTrue(StringUtil.isAscii("One Two"));
assertFalse(StringUtil.isAscii("🧔"));
assertFalse(StringUtil.isAscii("测试"));
assertFalse(StringUtil.isAscii("测试.com"));
}
@Test void isAsciiLetter() {
assertTrue(StringUtil.isAsciiLetter('a'));
assertTrue(StringUtil.isAsciiLetter('n'));
assertTrue(StringUtil.isAsciiLetter('z'));
assertTrue(StringUtil.isAsciiLetter('A'));
assertTrue(StringUtil.isAsciiLetter('N'));
assertTrue(StringUtil.isAsciiLetter('Z'));
assertFalse(StringUtil.isAsciiLetter(' '));
assertFalse(StringUtil.isAsciiLetter('-'));
assertFalse(StringUtil.isAsciiLetter('0'));
assertFalse(StringUtil.isAsciiLetter('ß'));
assertFalse(StringUtil.isAsciiLetter('Ě'));
}
@Test void isDigit() {
assertTrue(StringUtil.isDigit('0'));
assertTrue(StringUtil.isDigit('1'));
assertTrue(StringUtil.isDigit('2'));
assertTrue(StringUtil.isDigit('3'));
assertTrue(StringUtil.isDigit('4'));
assertTrue(StringUtil.isDigit('5'));
assertTrue(StringUtil.isDigit('6'));
assertTrue(StringUtil.isDigit('7'));
assertTrue(StringUtil.isDigit('8'));
assertTrue(StringUtil.isDigit('9'));
assertFalse(StringUtil.isDigit('a'));
assertFalse(StringUtil.isDigit('A'));
assertFalse(StringUtil.isDigit('ä'));
assertFalse(StringUtil.isDigit('Ä'));
assertFalse(StringUtil.isDigit('١'));
assertFalse(StringUtil.isDigit('୳'));
}
@Test void isHexDigit() {
assertTrue(StringUtil.isHexDigit('0'));
assertTrue(StringUtil.isHexDigit('1'));
assertTrue(StringUtil.isHexDigit('2'));
assertTrue(StringUtil.isHexDigit('3'));
assertTrue(StringUtil.isHexDigit('4'));
assertTrue(StringUtil.isHexDigit('5'));
assertTrue(StringUtil.isHexDigit('6'));
assertTrue(StringUtil.isHexDigit('7'));
assertTrue(StringUtil.isHexDigit('8'));
assertTrue(StringUtil.isHexDigit('9'));
assertTrue(StringUtil.isHexDigit('a'));
assertTrue(StringUtil.isHexDigit('b'));
assertTrue(StringUtil.isHexDigit('c'));
assertTrue(StringUtil.isHexDigit('d'));
assertTrue(StringUtil.isHexDigit('e'));
assertTrue(StringUtil.isHexDigit('f'));
assertTrue(StringUtil.isHexDigit('A'));
assertTrue(StringUtil.isHexDigit('B'));
assertTrue(StringUtil.isHexDigit('C'));
assertTrue(StringUtil.isHexDigit('D'));
assertTrue(StringUtil.isHexDigit('E'));
assertTrue(StringUtil.isHexDigit('F'));
assertFalse(StringUtil.isHexDigit('g'));
assertFalse(StringUtil.isHexDigit('G'));
assertFalse(StringUtil.isHexDigit('ä'));
assertFalse(StringUtil.isHexDigit('Ä'));
assertFalse(StringUtil.isHexDigit('١'));
assertFalse(StringUtil.isHexDigit('୳'));
}
}
================================================
FILE: src/test/java/org/jsoup/nodes/AttributeTest.java
================================================
package org.jsoup.nodes;
import org.jsoup.Jsoup;
import org.jsoup.parser.ParseSettings;
import org.jsoup.parser.Parser;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class AttributeTest {
@Test
public void html() {
Attribute attr = new Attribute("key", "value &");
assertEquals("key=\"value &\"", attr.html());
assertEquals(attr.html(), attr.toString());
}
@Test
public void htmlWithLtAndGtInValue() {
Attribute attr = new Attribute("key", "");
assertEquals("key=\"<value>\"", attr.html());
}
@Test public void testWithSupplementaryCharacterInAttributeKeyAndValue() {
String s = new String(Character.toChars(135361));
Attribute attr = new Attribute(s, "A" + s + "B");
assertEquals(s + "=\"A" + s + "B\"", attr.html());
assertEquals(attr.html(), attr.toString());
}
@Test public void validatesKeysNotEmpty() {
assertThrows(IllegalArgumentException.class, () -> new Attribute(" ", "Check"));
}
@Test public void validatesKeysNotEmptyViaSet() {
assertThrows(IllegalArgumentException.class, () -> {
Attribute attr = new Attribute("One", "Check");
attr.setKey(" ");
});
}
@Test public void booleanAttributesAreEmptyStringValues() {
Document doc = Jsoup.parse("");
Attributes attributes = doc.body().child(0).attributes();
assertEquals("", attributes.get("hidden"));
Attribute first = attributes.iterator().next();
assertEquals("hidden", first.getKey());
assertEquals("", first.getValue());
assertFalse(first.hasDeclaredValue());
assertTrue(Attribute.isBooleanAttribute(first.getKey()));
}
@Test public void settersOnOrphanAttribute() {
Attribute attr = new Attribute("one", "two");
attr.setKey("three");
String oldVal = attr.setValue("four");
assertEquals("two", oldVal);
assertEquals("three", attr.getKey());
assertEquals("four", attr.getValue());
assertNull(attr.parent);
}
@Test void settersAfterParentRemoval() {
// tests key and value set on a retained attribute after disconnected from parent
Attributes attrs = new Attributes();
attrs.put("foo", "bar");
Attribute attr = attrs.attribute("foo");
assertNotNull(attr);
attrs.remove("foo");
assertEquals("foo", attr.getKey());
assertEquals("bar", attr.getValue());
attr.setKey("new");
attr.setValue("newer");
assertEquals("new", attr.getKey());
assertEquals("newer", attr.getValue());
}
@Test public void hasValue() {
Attribute a1 = new Attribute("one", "");
Attribute a2 = new Attribute("two", null);
Attribute a3 = new Attribute("thr", "thr");
assertTrue(a1.hasDeclaredValue());
assertFalse(a2.hasDeclaredValue());
assertTrue(a3.hasDeclaredValue());
}
@Test public void canSetValueToNull() {
Attribute attr = new Attribute("one", "val");
String oldVal = attr.setValue(null);
assertEquals("one", attr.html());
assertEquals("val", oldVal);
oldVal = attr.setValue("foo");
assertEquals("", oldVal); // string, not null
}
@Test void booleanAttributesAreNotCaseSensitive() {
// https://github.com/jhy/jsoup/issues/1656
assertTrue(Attribute.isBooleanAttribute("required"));
assertTrue(Attribute.isBooleanAttribute("REQUIRED"));
assertTrue(Attribute.isBooleanAttribute("rEQUIREd"));
assertFalse(Attribute.isBooleanAttribute("random string"));
String html = "
One";
Document doc = Jsoup.parse(html);
assertEquals("
One", doc.selectFirst("a").outerHtml());
Document doc2 = Jsoup.parse(html, Parser.htmlParser().settings(ParseSettings.preserveCase));
assertEquals("
One", doc2.selectFirst("a").outerHtml());
}
@Test void orphanNamespace() {
Attribute attr = new Attribute("one", "two");
assertEquals("", attr.namespace());
}
}
================================================
FILE: src/test/java/org/jsoup/nodes/AttributesTest.java
================================================
package org.jsoup.nodes;
import org.jsoup.Jsoup;
import org.junit.jupiter.api.Test;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import static org.junit.jupiter.api.Assertions.*;
/**
* Tests for Attributes.
*
* @author Jonathan Hedley
*/
public class AttributesTest {
@Test
public void html() {
Attributes a = new Attributes();
a.put("Tot", "a&p");
a.put("Hello", "There");
a.put("data-name", "Jsoup");
assertEquals(3, a.size());
assertTrue(a.hasKey("Tot"));
assertTrue(a.hasKey("Hello"));
assertTrue(a.hasKey("data-name"));
assertFalse(a.hasKey("tot"));
assertTrue(a.hasKeyIgnoreCase("tot"));
assertEquals("There", a.getIgnoreCase("hEllo"));
Map
dataset = a.dataset();
assertEquals(1, dataset.size());
assertEquals("Jsoup", dataset.get("name"));
assertEquals("", a.get("tot"));
assertEquals("a&p", a.get("Tot"));
assertEquals("a&p", a.getIgnoreCase("tot"));
assertEquals(" Tot=\"a&p\" Hello=\"There\" data-name=\"Jsoup\"", a.html());
assertEquals(a.html(), a.toString());
}
@Test
public void testIteratorRemovable() {
Attributes a = new Attributes();
a.put("Tot", "a&p");
a.put("Hello", "There");
a.put("data-name", "Jsoup");
assertTrue(a.hasKey("Tot"));
Iterator iterator = a.iterator();
Attribute attr = iterator.next();
assertEquals("Tot", attr.getKey());
iterator.remove();
assertEquals(2, a.size());
attr = iterator.next();
assertEquals("Hello", attr.getKey());
assertEquals("There", attr.getValue());
// make sure that's flowing to the underlying attributes object
assertEquals(2, a.size());
assertEquals("There", a.get("Hello"));
assertFalse(a.hasKey("Tot"));
}
@Test
public void testIteratorUpdateable() {
Attributes a = new Attributes();
a.put("Tot", "a&p");
a.put("Hello", "There");
assertFalse(a.hasKey("Foo"));
Iterator iterator = a.iterator();
Attribute attr = iterator.next();
attr.setKey("Foo");
attr = iterator.next();
attr.setKey("Bar");
attr.setValue("Qux");
assertEquals("a&p", a.get("Foo"));
assertEquals("Qux", a.get("Bar"));
assertFalse(a.hasKey("Tot"));
assertFalse(a.hasKey("Hello"));
}
@Test public void testIteratorHasNext() {
Attributes a = new Attributes();
a.put("Tot", "1");
a.put("Hello", "2");
a.put("data-name", "3");
int seen = 0;
for (Attribute attribute : a) {
seen++;
assertEquals(String.valueOf(seen), attribute.getValue());
}
assertEquals(3, seen);
}
@Test
public void testIterator() {
Attributes a = new Attributes();
String[][] datas = {{"Tot", "raul"},
{"Hello", "pismuth"},
{"data-name", "Jsoup"}};
for (String[] atts : datas) {
a.put(atts[0], atts[1]);
}
Iterator iterator = a.iterator();
assertTrue(iterator.hasNext());
int i = 0;
for (Attribute attribute : a) {
assertEquals(datas[i][0], attribute.getKey());
assertEquals(datas[i][1], attribute.getValue());
i++;
}
assertEquals(datas.length, i);
}
@Test
public void testIteratorSkipsInternal() {
Attributes a = new Attributes();
a.put("One", "One");
a.put(Attributes.internalKey("baseUri"), "example.com");
a.put("Two", "Two");
a.put(Attributes.internalKey("another"), "example.com");
Iterator it = a.iterator();
assertTrue(it.hasNext());
assertEquals("One", it.next().getKey());
assertTrue(it.hasNext());
assertEquals("Two", it.next().getKey());
assertFalse(it.hasNext());
int seen = 0;
for (Attribute attribute : a) {
seen++;
}
assertEquals(2, seen);
}
@Test void iteratorThrows() {
Attributes attrs = new Attributes();
attrs.put("One", "one").put("Two", "two");
Iterator it = attrs.iterator();
int seen = 0;
while (it.hasNext()) {
it.next();
seen++;
}
assertFalse(it.hasNext());
assertEquals(2, seen);
boolean threw = false;
try {
Attribute next = it.next();
assertNotNull(next); // not hit
} catch (NoSuchElementException e) {
threw = true;
}
assertTrue(threw);
}
@Test
public void testListSkipsInternal() {
Attributes a = new Attributes();
a.put("One", "One");
a.put(Attributes.internalKey("baseUri"), "example.com");
a.put("Two", "Two");
a.put(Attributes.internalKey("another"), "example.com");
List attributes = a.asList();
assertEquals(2, attributes.size());
assertEquals("One", attributes.get(0).getKey());
assertEquals("Two", attributes.get(1). getKey());
}
@Test public void htmlSkipsInternals() {
Attributes a = new Attributes();
a.put("One", "One");
a.put(Attributes.internalKey("baseUri"), "example.com");
a.put("Two", "Two");
a.put(Attributes.internalKey("another"), "example.com");
assertEquals(" One=\"One\" Two=\"Two\"", a.html());
}
@Test
public void testIteratorEmpty() {
Attributes a = new Attributes();
Iterator iterator = a.iterator();
assertFalse(iterator.hasNext());
}
@Test
public void testIteratorRemove() {
String html = "