[ { "path": "AO3/__init__.py", "content": "from . import extra, utils\r\nfrom .chapters import Chapter\r\nfrom .comments import Comment\r\nfrom .search import Search\r\nfrom .series import Series\r\nfrom .session import GuestSession, Session\r\nfrom .users import User\r\nfrom .works import Work\r\n\r\nVERSION = \"2.3.0\"\r\n" }, { "path": "AO3/chapters.py", "content": "from functools import cached_property\n\nimport bs4\nfrom bs4 import BeautifulSoup\n\nfrom . import threadable, utils\nfrom .comments import Comment\nfrom .requester import requester\nfrom .users import User\n\n\nclass Chapter:\n \"\"\"\n AO3 chapter object\n \"\"\"\n \n def __init__(self, chapterid, work, session=None, load=True):\n self._session = session\n self._work = work\n self.id = chapterid\n self._soup = None\n if load:\n self.reload()\n \n def __repr__(self):\n if self.id is None:\n return f\"Chapter [ONESHOT] from [{self.work}]\"\n try:\n return f\"\"\n except:\n return f\"\"\n \n def __eq__(self, other):\n return isinstance(other, __class__) and other.id == self.id\n \n def __getstate__(self):\n d = {}\n for attr in self.__dict__:\n if isinstance(self.__dict__[attr], BeautifulSoup):\n d[attr] = (self.__dict__[attr].encode(), True)\n else:\n d[attr] = (self.__dict__[attr], False)\n return d\n \n def __setstate__(self, d):\n for attr in d:\n value, issoup = d[attr]\n if issoup:\n self.__dict__[attr] = BeautifulSoup(value, \"lxml\")\n else:\n self.__dict__[attr] = value\n \n def set_session(self, session):\n \"\"\"Sets the session used to make requests for this chapter\n\n Args:\n session (AO3.Session/AO3.GuestSession): session object\n \"\"\"\n \n self._session = session \n \n @threadable.threadable\n def reload(self):\n \"\"\"\n Loads information about this chapter.\n This function is threadable.\n \"\"\"\n from .works import Work\n \n for attr in self.__class__.__dict__:\n if isinstance(getattr(self.__class__, attr), cached_property):\n if attr in self.__dict__:\n delattr(self, attr)\n \n if self.work is None:\n soup = self.request(f\"https://archiveofourown.org/chapters/{self.id}?view_adult=true\")\n workid = soup.find(\"li\", {\"class\": \"chapter entire\"})\n if workid is None:\n raise utils.InvalidIdError(\"Cannot find work\")\n self._work = Work(utils.workid_from_url(workid.a[\"href\"]))\n else:\n self.work.reload()\n \n for chapter in self.work.chapters:\n if chapter == self:\n self._soup = chapter._soup\n \n @threadable.threadable\n def comment(self, comment_text, email=\"\", name=\"\", pseud=None):\n \"\"\"Leaves a comment on this chapter.\n This function is threadable.\n\n Args:\n comment_text (str): Comment text\n\n Raises:\n utils.UnloadedError: Couldn't load chapters\n utils.AuthError: Invalid session\n\n Returns:\n requests.models.Response: Response object\n \"\"\"\n \n if self.id is None:\n return self._work.comment(comment_text, email, name, pseud)\n \n if not self.loaded:\n raise utils.UnloadedError(\"Chapter isn't loaded. Have you tried calling Chapter.reload()?\")\n \n if self._session is None:\n raise utils.AuthError(\"Invalid session\")\n \n if self.id is not None:\n return utils.comment(self, comment_text, self._session, False, email=email, name=name, pseud=pseud)\n \n def get_comments(self, maximum=None):\n \"\"\"Returns a list of all threads of comments in the chapter. This operation can take a very long time.\n Because of that, it is recomended that you set a maximum number of comments. \n Duration: ~ (0.13 * n_comments) seconds or 2.9 seconds per comment page\n\n Args:\n maximum (int, optional): Maximum number of comments to be returned. None -> No maximum\n\n Raises:\n ValueError: Invalid chapter number\n IndexError: Invalid chapter number\n utils.UnloadedError: Chapter isn't loaded\n\n Returns:\n list: List of comments\n \"\"\"\n \n if self.id is None:\n return self._work.get_comments(maximum=maximum)\n \n if not self.loaded:\n raise utils.UnloadedError(\"Chapter isn't loaded. Have you tried calling Chapter.reload()?\")\n \n url = f\"https://archiveofourown.org/chapters/{self.id}?page=%d&show_comments=true&view_adult=true\"\n soup = self.request(url%1)\n \n pages = 0\n div = soup.find(\"div\", {\"id\": \"comments_placeholder\"})\n ol = div.find(\"ol\", {\"class\": \"pagination actions\"})\n if ol is None:\n pages = 1\n else:\n for li in ol.findAll(\"li\"):\n if li.getText().isdigit():\n pages = int(li.getText()) \n \n comments = []\n for page in range(pages):\n if page != 0:\n soup = self.request(url%(page+1))\n ol = soup.find(\"ol\", {\"class\": \"thread\"})\n for li in ol.findAll(\"li\", {\"role\": \"article\"}, recursive=False):\n if maximum is not None and len(comments) >= maximum:\n return comments\n id_ = int(li.attrs[\"id\"][8:])\n \n header = li.find(\"h4\", {\"class\": (\"heading\", \"byline\")})\n if header is None:\n author = None\n else:\n author = User(str(header.a.text), self._session, False)\n \n if li.blockquote is not None:\n text = li.blockquote.getText()\n else:\n text = \"\" \n \n comment = Comment(id_, self, session=self._session, load=False) \n setattr(comment, \"authenticity_token\", self.authenticity_token)\n setattr(comment, \"author\", author)\n setattr(comment, \"text\", text)\n comment._thread = None\n comments.append(comment)\n return comments\n \n def get_images(self):\n \"\"\"Gets all images from this work\n\n Raises:\n utils.UnloadedError: Raises this error if the chapter isn't loaded\n\n Returns:\n tuple: Pairs of image urls and the paragraph number\n \"\"\"\n \n div = self._soup.find(\"div\", {\"class\": \"userstuff\"})\n images = []\n line = 0\n for p in div.findAll(\"p\"):\n line += 1\n for img in p.findAll(\"img\"):\n if \"src\" in img.attrs:\n images.append((img.attrs[\"src\"], line))\n return tuple(images)\n \n @property\n def loaded(self):\n \"\"\"Returns True if this chapter has been loaded\"\"\"\n return self._soup is not None\n \n @property\n def authenticity_token(self):\n \"\"\"Token used to take actions that involve this work\"\"\"\n return self.work.authenticity_token\n \n @property\n def work(self):\n \"\"\"Work this chapter is a part of\"\"\"\n return self._work\n \n @cached_property\n def text(self):\n \"\"\"This chapter's text\"\"\"\n text = \"\"\n if self.id is not None:\n div = self._soup.find(\"div\", {\"role\": \"article\"})\n else:\n div = self._soup\n for p in div.findAll((\"p\", \"center\")):\n text += p.getText().replace(\"\\n\", \"\") + \"\\n\"\n if isinstance(p.next_sibling, bs4.element.NavigableString):\n text += str(p.next_sibling)\n return text\n\n @cached_property\n def title(self):\n \"\"\"This chapter's title\"\"\"\n if self.id is None:\n return self.work.title\n preface_group = self._soup.find(\"div\", {\"class\": (\"chapter\", \"preface\", \"group\")})\n if preface_group is None:\n return str(self.number)\n title = preface_group.find(\"h3\", {\"class\": \"title\"})\n if title is None:\n return str(self.number)\n return tuple(title.strings)[-1].strip()[2:]\n \n @cached_property\n def number(self):\n \"\"\"This chapter's number\"\"\"\n if self.id is None:\n return 1\n return int(self._soup[\"id\"].split(\"-\")[-1])\n \n @cached_property\n def words(self):\n \"\"\"Number of words from this chapter\"\"\"\n return utils.word_count(self.text)\n \n @cached_property\n def summary(self):\n \"\"\"Text from this chapter's summary\"\"\"\n notes = self._soup.find(\"div\", {\"id\": \"summary\"})\n if notes is None:\n return \"\"\n text = \"\"\n for p in notes.findAll(\"p\"):\n text += p.getText() + \"\\n\"\n return text\n\n @cached_property\n def start_notes(self):\n \"\"\"Text from this chapter's start notes\"\"\"\n notes = self._soup.find(\"div\", {\"id\": \"notes\"})\n if notes is None:\n return \"\"\n text = \"\"\n for p in notes.findAll(\"p\"):\n text += p.getText().strip() + \"\\n\"\n return text\n\n @cached_property\n def end_notes(self):\n \"\"\"Text from this chapter's end notes\"\"\"\n notes = self._soup.find(\"div\", {\"id\": f\"chapter_{self.number}_endnotes\"})\n if notes is None:\n return \"\"\n text = \"\"\n for p in notes.findAll(\"p\"):\n text += p.getText() + \"\\n\"\n return text\n \n @cached_property\n def url(self):\n \"\"\"Returns the URL to this chapter\n\n Returns:\n str: chapter URL\n \"\"\"\n\n return f\"https://archiveofourown.org/works/{self._work.id}/chapters/{self.id}\"\n\n def request(self, url):\n \"\"\"Request a web page and return a BeautifulSoup object.\n\n Args:\n url (str): Url to request\n\n Returns:\n bs4.BeautifulSoup: BeautifulSoup object representing the requested page's html\n \"\"\"\n\n req = self.get(url)\n soup = BeautifulSoup(req.content, \"lxml\")\n return soup\n \n def get(self, *args, **kwargs):\n \"\"\"Request a web page and return a Response object\"\"\" \n \n if self._session is None:\n req = requester.request(\"get\", *args, **kwargs)\n else:\n req = requester.request(\"get\", *args, **kwargs, session=self._session.session)\n if req.status_code == 429:\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\n return req\n" }, { "path": "AO3/comments.py", "content": "from functools import cached_property\n\nfrom bs4 import BeautifulSoup\n\nfrom . import threadable, utils\nfrom .requester import requester\nfrom .users import User\n\n\nclass Comment:\n \"\"\"\n AO3 comment object\n \"\"\"\n \n def __init__(self, comment_id, parent=None, parent_comment=None, session=None, load=True):\n \"\"\"Creates a new AO3 comment object\n\n Args:\n comment_id (int/str): Comment ID\n parent (Work/Chapter, optional): Parent object (where the comment is posted). Defaults to None.\n parent_comment (Comment, optional): Parent comment. Defaults to None.\n session (Session/GuestSession, optional): Session object\n load (boolean, optional): If true, the comment is loaded on initialization. Defaults to True.\n \"\"\"\n \n self.id = comment_id\n self.parent = parent\n self.parent_comment = parent_comment\n self.authenticity_token = None\n self._thread = None\n self._session = session\n self.__soup = None\n if load:\n self.reload()\n \n def __repr__(self):\n return f\"\"\n \n @property\n def _soup(self):\n if self.__soup is None:\n if self.parent_comment is None:\n return None\n return self.parent_comment._soup\n return self.__soup\n \n @property\n def first_parent_comment(self):\n if self.parent_comment is None:\n return self\n else:\n return self.parent_comment.first_parent_comment\n \n @property\n def fullwork(self):\n from .works import Work\n if self.parent is None:\n return None\n return isinstance(self.parent, Work)\n \n @cached_property\n def author(self):\n \"\"\"Comment author\"\"\"\n li = self._soup.find(\"li\", {\"id\": f\"comment_{self.id}\"})\n header = li.find(\"h4\", {\"class\": (\"heading\", \"byline\")})\n if header is None:\n author = None\n else:\n author = User(str(header.a.text), self._session, False)\n return author\n \n @cached_property\n def text(self):\n \"\"\"Comment text\"\"\"\n li = self._soup.find(\"li\", {\"id\": f\"comment_{self.id}\"})\n if li.blockquote is not None:\n text = li.blockquote.getText()\n else:\n text = \"\"\n return text\n \n def get_thread(self):\n \"\"\"Returns all the replies to this comment, and all subsequent replies recursively.\n Also loads any parent comments this comment might have.\n\n Raises:\n utils.InvalidIdError: The specified comment_id was invalid\n\n Returns:\n list: Thread\n \"\"\"\n \n if self._thread is not None:\n return self._thread\n else:\n if self._soup is None:\n self.reload()\n \n nav = self._soup.find(\"ul\", {\"id\": f\"navigation_for_comment_{self.id}\"})\n for li in nav.findAll(\"li\"):\n if li.getText() == \"\\nParent Thread\\n\":\n id_ = int(li.a[\"href\"].split(\"/\")[-1])\n parent = Comment(id_, session=self._session)\n for comment in parent.get_thread_iterator():\n if comment.id == self.id:\n index = comment.parent_comment._thread.index(comment)\n comment.parent_comment._thread.pop(index)\n comment.parent_comment._thread.insert(index, self)\n self._thread = comment._thread\n self.parent_comment = comment.parent_comment\n del comment\n return self._thread\n \n thread = self._soup.find(\"ol\", {\"class\": \"thread\"})\n if thread is None:\n self._thread = []\n return self._thread\n \n self._get_thread(None, thread)\n \n if self._thread is None:\n self._thread = []\n return self._thread\n \n def _get_thread(self, parent, soup):\n comments = soup.findAll(\"li\", recursive=False)\n l = [self] if parent is None else []\n for comment in comments:\n if \"role\" in comment.attrs:\n id_ = int(comment.attrs[\"id\"][8:])\n c = Comment(id_, self.parent, session=self._session, load=False)\n c.authenticity_token = self.authenticity_token\n c._thread = []\n if parent is not None:\n c.parent_comment = parent\n if comment.blockquote is not None:\n text = comment.blockquote.getText()\n else:\n text = \"\"\n if comment.a is not None:\n author = User(comment.a.getText(), load=False)\n else:\n author = None\n setattr(c, \"text\", text)\n setattr(c, \"author\", author)\n l.append(c)\n else:\n c.parent_comment = self\n if comment.blockquote is not None:\n text = comment.blockquote.getText()\n else:\n text = \"\"\n if comment.a is not None:\n author = User(comment.a.getText(), load=False)\n else:\n author = None\n setattr(l[0], \"text\", text)\n setattr(l[0], \"author\", author)\n else:\n self._get_thread(l[-1], comment.ol)\n if parent is not None:\n parent._thread = l\n \n def get_thread_iterator(self):\n \"\"\"Returns a generator that allows you to iterate through the entire thread\n\n Returns:\n generator: The generator object\n \"\"\"\n \n return threadIterator(self)\n \n @threadable.threadable\n def reply(self, comment_text, email=\"\", name=\"\"):\n \"\"\"Replies to a comment.\n This function is threadable.\n\n Args:\n comment_text (str): Comment text\n email (str, optional): Email. Defaults to \"\".\n name (str, optional): Name. Defaults to \"\".\n\n Raises:\n utils.InvalidIdError: Invalid ID\n utils.UnexpectedResponseError: Unknown error\n utils.PseudoError: Couldn't find a valid pseudonym to post under\n utils.DuplicateCommentError: The comment you're trying to post was already posted\n ValueError: Invalid name/email\n ValueError: self.parent cannot be None\n\n Returns:\n requests.models.Response: Response object\n \"\"\"\n \n if self.parent is None:\n raise ValueError(\"self.parent cannot be 'None'\")\n return utils.comment(self.parent, comment_text, self._session, self.fullwork, self.id, email, name)\n \n @threadable.threadable\n def reload(self):\n \"\"\"Loads all comment properties\n This function is threadable.\n \"\"\"\n from .works import Work\n \n for attr in self.__class__.__dict__:\n if isinstance(getattr(self.__class__, attr), cached_property):\n if attr in self.__dict__:\n delattr(self, attr)\n \n req = self.get(f\"https://archiveofourown.org/comments/{self.id}\")\n self.__soup = BeautifulSoup(req.content, features=\"lxml\")\n \n token = self.__soup.find(\"meta\", {\"name\": \"csrf-token\"})\n self.authenticity_token = token[\"content\"]\n \n self._thread = None\n \n li = self._soup.find(\"li\", {\"id\": f\"comment_{self.id}\"})\n \n reply_link = li.find(\"li\", {\"id\": f\"add_comment_reply_link_{self.id}\"})\n \n if self.parent is None:\n if reply_link is not None:\n fields = [field.split(\"=\") for field in reply_link.a[\"href\"].split(\"?\")[-1].split(\"&\")]\n for key, value in fields:\n if key == \"chapter_id\":\n self.parent = int(value)\n break\n self.parent_comment = None\n \n @threadable.threadable\n def delete(self):\n \"\"\"Deletes this comment.\n This function is threadable.\n \n Raises:\n PermissionError: You don't have permission to delete the comment\n utils.AuthError: Invalid auth token\n utils.UnexpectedResponseError: Unknown error\n \"\"\"\n \n utils.delete_comment(self, self._session)\n \n def get(self, *args, **kwargs):\n \"\"\"Request a web page and return a Response object\"\"\" \n \n if self._session is None:\n req = requester.request(\"get\", *args, **kwargs)\n else:\n req = requester.request(\"get\", *args, **kwargs, session=self._session.session)\n if req.status_code == 429:\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\n return req\n \ndef threadIterator(comment):\n if comment.get_thread() is None or len(comment.get_thread()) == 0:\n yield comment\n else:\n for c in comment.get_thread():\n yield c\n for sub in threadIterator(c):\n if c != sub:\n yield sub\n" }, { "path": "AO3/common.py", "content": "import datetime\n\nfrom . import utils\n\n\ndef __setifnotnone(obj, attr, value):\n if value is not None:\n setattr(obj, attr, value)\n\ndef get_work_from_banner(work):\n #* These imports need to be here to prevent circular imports\n #* (series.py would requite common.py and vice-versa)\n from .series import Series\n from .users import User\n from .works import Work\n \n authors = []\n try:\n for a in work.h4.find_all(\"a\"):\n if 'rel' in a.attrs.keys():\n if \"author\" in a['rel']:\n authors.append(User(a.string, load=False))\n elif a.attrs[\"href\"].startswith(\"/works\"):\n workname = a.string\n workid = utils.workid_from_url(a['href'])\n except AttributeError:\n pass\n \n new = Work(workid, load=False)\n\n fandoms = []\n try:\n for a in work.find(\"h5\", {\"class\": \"fandoms\"}).find_all(\"a\"):\n fandoms.append(a.string)\n except AttributeError:\n pass\n\n warnings = []\n relationships = []\n characters = []\n freeforms = []\n try:\n for a in work.find(attrs={\"class\": \"tags\"}).find_all(\"li\"):\n if \"warnings\" in a['class']:\n warnings.append(a.text)\n elif \"relationships\" in a['class']:\n relationships.append(a.text)\n elif \"characters\" in a['class']:\n characters.append(a.text)\n elif \"freeforms\" in a['class']:\n freeforms.append(a.text)\n except AttributeError:\n pass\n\n reqtags = work.find(attrs={\"class\": \"required-tags\"})\n if reqtags is not None:\n rating = reqtags.find(attrs={\"class\": \"rating\"})\n if rating is not None:\n rating = rating.text\n categories = reqtags.find(attrs={\"class\": \"category\"})\n if categories is not None:\n categories = categories.text.split(\", \")\n else:\n rating = categories = None\n\n summary = work.find(attrs={\"class\": \"userstuff summary\"})\n if summary is not None:\n summary = summary.text\n\n series = []\n series_list = work.find(attrs={\"class\": \"series\"})\n if series_list is not None:\n for a in series_list.find_all(\"a\"):\n seriesid = int(a.attrs['href'].split(\"/\")[-1])\n seriesname = a.text\n s = Series(seriesid, load=False)\n setattr(s, \"name\", seriesname)\n series.append(s)\n\n stats = work.find(attrs={\"class\": \"stats\"})\n if stats is not None:\n language = stats.find(\"dd\", {\"class\": \"language\"})\n if language is not None:\n language = language.text\n words = stats.find(\"dd\", {\"class\": \"words\"})\n if words is not None:\n words = words.text.replace(\",\", \"\")\n if words.isdigit(): words = int(words)\n else: words = None\n bookmarks = stats.find(\"dd\", {\"class\": \"bookmarks\"})\n if bookmarks is not None:\n bookmarks = bookmarks.text.replace(\",\", \"\")\n if bookmarks.isdigit(): bookmarks = int(bookmarks)\n else: bookmarks = None\n chapters = stats.find(\"dd\", {\"class\": \"chapters\"})\n if chapters is not None:\n chapters = chapters.text.split('/')[0].replace(\",\", \"\")\n if chapters.isdigit(): chapters = int(chapters)\n else: chapters = None\n expected_chapters = stats.find(\"dd\", {\"class\": \"chapters\"})\n if expected_chapters is not None:\n expected_chapters = expected_chapters.text.split('/')[-1].replace(\",\", \"\")\n if expected_chapters.isdigit(): expected_chapters = int(expected_chapters)\n else: expected_chapters = None\n hits = stats.find(\"dd\", {\"class\": \"hits\"})\n if hits is not None:\n hits = hits.text.replace(\",\", \"\")\n if hits.isdigit(): hits = int(hits)\n else: hits = None\n kudos = stats.find(\"dd\", {\"class\": \"kudos\"})\n if kudos is not None:\n kudos = kudos.text.replace(\",\", \"\")\n if kudos.isdigit(): kudos = int(kudos)\n else: kudos = None\n comments = stats.find(\"dd\", {\"class\": \"comments\"})\n if comments is not None:\n comments = comments.text.replace(\",\", \"\")\n if comments.isdigit(): comments = int(comments)\n else: comments = None\n restricted = work.find(\"img\", {\"title\": \"Restricted\"}) is not None\n if chapters is None:\n complete = None\n else:\n complete = chapters == expected_chapters\n else:\n language = words = bookmarks = chapters = expected_chapters = hits = restricted = complete = None\n\n date = work.find(\"p\", {\"class\": \"datetime\"})\n if date is None:\n date_updated = None\n else:\n date_updated = datetime.datetime.strptime(date.getText(), \"%d %b %Y\")\n\n __setifnotnone(new, \"authors\", authors)\n __setifnotnone(new, \"bookmarks\", bookmarks)\n __setifnotnone(new, \"categories\", categories)\n __setifnotnone(new, \"nchapters\", chapters)\n __setifnotnone(new, \"characters\", characters)\n __setifnotnone(new, \"complete\", complete)\n __setifnotnone(new, \"date_updated\", date_updated)\n __setifnotnone(new, \"expected_chapters\", expected_chapters)\n __setifnotnone(new, \"fandoms\", fandoms)\n __setifnotnone(new, \"hits\", hits)\n __setifnotnone(new, \"comments\", comments)\n __setifnotnone(new, \"kudos\", kudos)\n __setifnotnone(new, \"language\", language)\n __setifnotnone(new, \"rating\", rating)\n __setifnotnone(new, \"relationships\", relationships)\n __setifnotnone(new, \"restricted\", restricted)\n __setifnotnone(new, \"series\", series)\n __setifnotnone(new, \"summary\", summary)\n __setifnotnone(new, \"tags\", freeforms)\n __setifnotnone(new, \"title\", workname)\n __setifnotnone(new, \"warnings\", warnings)\n __setifnotnone(new, \"words\", words)\n \n return new\n\ndef url_join(base, *args):\n result = base\n for arg in args:\n if len(result) > 0 and not result[-1] == \"/\":\n result += \"/\"\n if len(arg) > 0 and arg[0] != \"/\":\n result += arg\n else:\n result += arg[1:]\n return result" }, { "path": "AO3/extra.py", "content": "import functools\nimport os\nimport pathlib\nimport pickle\n\nfrom bs4 import BeautifulSoup\n\nfrom . import threadable, utils\nfrom .requester import requester\n\n\ndef _download_languages():\n path = os.path.dirname(__file__)\n languages = []\n try:\n rsrc_path = os.path.join(path, \"resources\")\n if not os.path.isdir(rsrc_path):\n os.mkdir(rsrc_path)\n language_path = os.path.join(rsrc_path, \"languages\")\n if not os.path.isdir(language_path):\n os.mkdir(language_path)\n url = \"https://archiveofourown.org/languages\"\n print(f\"Downloading from {url}\")\n req = requester.request(\"get\", url)\n soup = BeautifulSoup(req.content, \"lxml\")\n for dt in soup.find(\"dl\", {\"class\": \"language index group\"}).findAll(\"dt\"):\n if dt.a is not None: \n alias = dt.a.attrs[\"href\"].split(\"/\")[-1]\n else:\n alias = None\n languages.append((dt.getText(), alias))\n with open(f\"{os.path.join(language_path, 'languages')}.pkl\", \"wb\") as file:\n pickle.dump(languages, file)\n except AttributeError:\n raise utils.UnexpectedResponseError(\"Couldn't download the desired resource. Do you have the latest version of ao3-api?\")\n print(f\"Download complete ({len(languages)} languages)\")\n\ndef _download_fandom(fandom_key, name):\n path = os.path.dirname(__file__)\n fandoms = []\n try:\n rsrc_path = os.path.join(path, \"resources\")\n if not os.path.isdir(rsrc_path):\n os.mkdir(rsrc_path)\n fandom_path = os.path.join(rsrc_path, \"fandoms\")\n if not os.path.isdir(fandom_path):\n os.mkdir(fandom_path)\n url = f\"https://archiveofourown.org/media/{fandom_key}/fandoms\"\n print(f\"Downloading from {url}\")\n req = requester.request(\"get\", url)\n soup = BeautifulSoup(req.content, \"lxml\")\n for fandom in soup.find(\"ol\", {\"class\": \"alphabet fandom index group\"}).findAll(\"a\", {\"class\": \"tag\"}):\n fandoms.append(fandom.getText())\n with open(f\"{os.path.join(fandom_path, name)}.pkl\", \"wb\") as file:\n pickle.dump(fandoms, file)\n except AttributeError:\n raise utils.UnexpectedResponseError(\"Couldn't download the desired resource. Do you have the latest version of ao3-api?\")\n print(f\"Download complete ({len(fandoms)} fandoms)\")\n \n\n_FANDOM_RESOURCES = {\n \"anime_manga_fandoms\": functools.partial(\n _download_fandom, \n \"Anime%20*a*%20Manga\", \n \"anime_manga_fandoms\"),\n \"books_literature_fandoms\": functools.partial(\n _download_fandom, \n \"Books%20*a*%20Literature\", \n \"books_literature_fandoms\"),\n \"cartoons_comics_graphicnovels_fandoms\": functools.partial(\n _download_fandom, \n \"Cartoons%20*a*%20Comics%20*a*%20Graphic%20Novels\", \n \"cartoons_comics_graphicnovels_fandoms\"),\n \"celebrities_real_people_fandoms\": functools.partial(\n _download_fandom, \n \"Celebrities%20*a*%20Real%20People\", \n \"celebrities_real_people_fandoms\"),\n \"movies_fandoms\": functools.partial(\n _download_fandom, \n \"Movies\", \n \"movies_fandoms\"),\n \"music_bands_fandoms\": functools.partial(\n _download_fandom, \n \"Music%20*a*%20Bands\", \n \"music_bands_fandoms\"),\n \"other_media_fandoms\": functools.partial(\n _download_fandom, \n \"Other%20Media\", \n \"other_media_fandoms\"),\n \"theater_fandoms\": functools.partial(\n _download_fandom, \n \"Theater\", \n \"theater_fandoms\"),\n \"tvshows_fandoms\": functools.partial(\n _download_fandom, \n \"TV%20Shows\", \n \"tvshows_fandoms\"),\n \"videogames_fandoms\": functools.partial(\n _download_fandom, \n \"Video%20Games\", \n \"videogames_fandoms\"),\n \"uncategorized_fandoms\": functools.partial(\n _download_fandom, \n \"Uncategorized%20Fandoms\", \n \"uncategorized_fandoms\")\n}\n\n_LANGUAGE_RESOURCES = {\n \"languages\": _download_languages\n}\n\n_RESOURCE_DICTS = [(\"fandoms\", _FANDOM_RESOURCES),\n (\"languages\", _LANGUAGE_RESOURCES)]\n\n@threadable.threadable\ndef download(resource):\n \"\"\"Downloads the specified resource.\n This function is threadable.\n\n Args:\n resource (str): Resource name\n\n Raises:\n KeyError: Invalid resource\n \"\"\"\n \n for _, resource_dict in _RESOURCE_DICTS:\n if resource in resource_dict:\n resource_dict[resource]()\n return\n raise KeyError(f\"'{resource}' is not a valid resource\")\n\ndef get_resources():\n \"\"\"Returns a list of every resource available for download\"\"\"\n \n d = {}\n for name, resource_dict in _RESOURCE_DICTS:\n d[name] = list(resource_dict.keys())\n return d\n\ndef has_resource(resource):\n \"\"\"Returns True if resource was already download, False otherwise\"\"\"\n path = os.path.join(os.path.dirname(__file__), \"resources\")\n return len(list(pathlib.Path(path).rglob(resource+\".pkl\"))) > 0\n\n@threadable.threadable\ndef download_all(redownload=False):\n \"\"\"Downloads every available resource.\n This function is threadable.\"\"\"\n \n types = get_resources()\n for rsrc_type in types:\n for rsrc in types[rsrc_type]:\n if redownload or not has_resource(rsrc):\n download(rsrc)\n\n@threadable.threadable \ndef download_all_threaded(redownload=False):\n \"\"\"Downloads every available resource in parallel (about ~3.7x faster).\n This function is threadable.\"\"\"\n \n threads = []\n types = get_resources()\n for rsrc_type in types:\n for rsrc in types[rsrc_type]:\n if redownload or not has_resource(rsrc):\n threads.append(download(rsrc, threaded=True))\n for thread in threads:\n thread.join()\n" }, { "path": "AO3/requester.py", "content": "import threading\nimport time\n\nimport requests\n\n\nclass Requester:\n \"\"\"Requester object\"\"\"\n \n def __init__(self, rqtw=-1, timew=60):\n \"\"\"Limits the request rate to prevent HTTP 429 (rate limiting) responses.\n 12 request per minute seems to be the limit.\n\n Args:\n rqm (int, optional): Maximum requests per time window (-1 -> no limit). Defaults to -1.\n timew (int, optional): Time window (seconds). Defaults to 60.\n \"\"\"\n \n self._requests = []\n self._rqtw = rqtw\n self._timew = timew\n self._lock = threading.Lock()\n self.total = 0\n \n def setRQTW(self, value):\n self._rqtw = value\n \n def setTimeW(self, value):\n self._timew = value\n\n def request(self, *args, **kwargs):\n \"\"\"Requests a web page once enough time has passed since the last request\n \n Args:\n session(requests.Session, optional): Session object to request with\n\n Returns:\n requests.Response: Response object\n \"\"\"\n \n # We've made a bunch of requests, time to rate limit?\n if self._rqtw != -1:\n with self._lock:\n if len(self._requests) >= self._rqtw:\n t = time.time()\n # Reduce list to only requests made within the current time window\n while len(self._requests):\n if t-self._requests[0] >= self._timew:\n self._requests.pop(0) # Older than window, forget about it\n else:\n break # Inside window, the rest of them must be too\n # Have we used up all available requests within our window?\n if len(self._requests) >= self._rqtw: # Yes\n # Wait until the oldest request exits the window, giving us a slot for the new one\n time.sleep(self._requests[0] + self._timew - t)\n # Now outside window, drop it\n self._requests.pop(0)\n \n if self._rqtw != -1:\n self._requests.append(time.time())\n self.total += 1\n \n if \"session\" in kwargs:\n sess = kwargs[\"session\"]\n del kwargs[\"session\"]\n req = sess.request(*args, **kwargs)\n else:\n req = requests.request(*args, **kwargs)\n \n return req\n\nrequester = Requester()" }, { "path": "AO3/search.py", "content": "from math import ceil\r\n\r\nfrom bs4 import BeautifulSoup\r\n\r\nfrom . import threadable, utils\r\nfrom .common import get_work_from_banner\r\nfrom .requester import requester\r\nfrom .series import Series\r\nfrom .users import User\r\nfrom .works import Work\r\n\r\nDEFAULT = \"_score\"\r\nBEST_MATCH = \"_score\"\r\nAUTHOR = \"authors_to_sort_on\"\r\nTITLE = \"title_to_sort_on\"\r\nDATE_POSTED = \"created_at\"\r\nDATE_UPDATED = \"revised_at\"\r\nWORD_COUNT = \"word_count\"\r\nRATING = \"rating_ids\"\r\nHITS = \"hits\"\r\nBOOKMARKS = \"bookmarks_count\"\r\nCOMMENTS = \"comments_count\"\r\nKUDOS = \"kudos_count\"\r\n\r\nDESCENDING = \"desc\"\r\nASCENDING = \"asc\"\r\n\r\n\r\nclass Search:\r\n def __init__(\r\n self,\r\n any_field=\"\",\r\n title=\"\",\r\n author=\"\",\r\n single_chapter=False,\r\n word_count=None,\r\n language=\"\",\r\n fandoms=\"\",\r\n rating=None,\r\n hits=None,\r\n kudos=None,\r\n crossovers=None,\r\n bookmarks=None,\r\n excluded_tags=\"\",\r\n comments=None,\r\n completion_status=None,\r\n page=1,\r\n sort_column=\"\",\r\n sort_direction=\"\",\r\n revised_at=\"\",\r\n characters=\"\",\r\n relationships=\"\",\r\n tags=\"\",\r\n session=None):\r\n\r\n self.any_field = any_field\r\n self.title = title\r\n self.author = author\r\n self.single_chapter = single_chapter\r\n self.word_count = word_count\r\n self.language = language\r\n self.fandoms = fandoms\r\n self.characters = characters\r\n self.relationships = relationships\r\n self.tags = tags\r\n self.rating = rating\r\n self.hits = hits\r\n self.kudos = kudos\r\n self.crossovers = crossovers\r\n self.bookmarks = bookmarks\r\n self.excluded_tags = excluded_tags\r\n self.comments = comments\r\n self.completion_status = completion_status\r\n self.page = page\r\n self.sort_column = sort_column\r\n self.sort_direction = sort_direction\r\n self.revised_at = revised_at\r\n \r\n self.session = session\r\n\r\n self.results = None\r\n self.pages = 0\r\n self.total_results = 0\r\n\r\n @threadable.threadable\r\n def update(self):\r\n \"\"\"Sends a request to the AO3 website with the defined search parameters, and updates all info.\r\n This function is threadable.\r\n \"\"\"\r\n\r\n soup = search(\r\n self.any_field, self.title, self.author, self.single_chapter,\r\n self.word_count, self.language, self.fandoms, self.rating, self.hits,\r\n self.kudos, self.crossovers, self.bookmarks, self.excluded_tags, self.comments, self.completion_status, self.page,\r\n self.sort_column, self.sort_direction, self.revised_at, self.session,\r\n self.characters, self.relationships, self.tags)\r\n\r\n results = soup.find(\"ol\", {\"class\": (\"work\", \"index\", \"group\")})\r\n if results is None and soup.find(\"p\", text=\"No results found. You may want to edit your search to make it less specific.\") is not None:\r\n self.results = []\r\n self.total_results = 0\r\n self.pages = 0\r\n return\r\n\r\n works = []\r\n for work in results.find_all(\"li\", {\"role\": \"article\"}):\r\n if work.h4 is None:\r\n continue\r\n \r\n new = get_work_from_banner(work)\r\n new._session = self.session\r\n works.append(new)\r\n\r\n self.results = works\r\n maindiv = soup.find(\"div\", {\"class\": \"works-search region\", \"id\": \"main\"})\r\n self.total_results = int(maindiv.find(\"h3\", {\"class\": \"heading\"}).getText().replace(',','').replace('.','').strip().split(\" \")[0])\r\n self.pages = ceil(self.total_results / 20)\r\n\r\ndef search(\r\n any_field=\"\",\r\n title=\"\",\r\n author=\"\",\r\n single_chapter=False,\r\n word_count=None,\r\n language=\"\",\r\n fandoms=\"\",\r\n rating=None,\r\n hits=None,\r\n kudos=None,\r\n crossovers=None,\r\n bookmarks=None,\r\n excluded_tags=\"\",\r\n comments=None,\r\n completion_status=None,\r\n page=1,\r\n sort_column=\"\",\r\n sort_direction=\"\",\r\n revised_at=\"\",\r\n session=None,\r\n characters=\"\",\r\n relationships=\"\",\r\n tags=\"\"):\r\n \"\"\"Returns the results page for the search as a Soup object\r\n\r\n Args:\r\n any_field (str, optional): Generic search. Defaults to \"\".\r\n title (str, optional): Title of the work. Defaults to \"\".\r\n author (str, optional): Authors of the work. Defaults to \"\".\r\n single_chapter (bool, optional): Only include one-shots. Defaults to False.\r\n word_count (AO3.utils.Constraint, optional): Word count. Defaults to None.\r\n language (str, optional): Work language. Defaults to \"\".\r\n fandoms (str, optional): Fandoms included in the work. Defaults to \"\".\r\n characters (str, optional): Characters included in the work. Defaults to \"\".\r\n relationships (str, optional): Relationships included in the work. Defaults to \"\".\r\n tags (str, optional): Additional tags applied to the work. Defaults to \"\".\r\n rating (int, optional): Rating for the work. 9 for Not Rated, 10 for General Audiences, 11 for Teen And Up Audiences, 12 for Mature, 13 for Explicit. Defaults to None.\r\n hits (AO3.utils.Constraint, optional): Number of hits. Defaults to None.\r\n kudos (AO3.utils.Constraint, optional): Number of kudos. Defaults to None.\r\n crossovers (bool, optional): If specified, if false, exclude crossovers, if true, include only crossovers\r\n bookmarks (AO3.utils.Constraint, optional): Number of bookmarks. Defaults to None.\r\n excluded_tags (str, optional): Tags to exclude. Defaults to \"\".\r\n comments (AO3.utils.Constraint, optional): Number of comments. Defaults to None.\r\n page (int, optional): Page number. Defaults to 1.\r\n sort_column (str, optional): Which column to sort on. Defaults to \"\".\r\n sort_direction (str, optional): Which direction to sort. Defaults to \"\".\r\n revised_at (str, optional): Show works older / more recent than this date. Defaults to \"\".\r\n session (AO3.Session, optional): Session object. Defaults to None.\r\n\r\n Returns:\r\n bs4.BeautifulSoup: Search result's soup\r\n \"\"\"\r\n\r\n query = utils.Query()\r\n query.add_field(f\"work_search[query]={any_field if any_field != '' else ' '}\")\r\n if page != 1:\r\n query.add_field(f\"page={page}\")\r\n if title != \"\":\r\n query.add_field(f\"work_search[title]={title}\")\r\n if author != \"\":\r\n query.add_field(f\"work_search[creators]={author}\")\r\n if single_chapter:\r\n query.add_field(f\"work_search[single_chapter]=1\")\r\n if word_count is not None:\r\n query.add_field(f\"work_search[word_count]={word_count}\")\r\n if language != \"\":\r\n query.add_field(f\"work_search[language_id]={language}\")\r\n if fandoms != \"\":\r\n query.add_field(f\"work_search[fandom_names]={fandoms}\")\r\n if characters != \"\":\r\n query.add_field(f\"work_search[character_names]={characters}\")\r\n if relationships != \"\":\r\n query.add_field(f\"work_search[relationship_names]={relationships}\")\r\n if tags != \"\":\r\n query.add_field(f\"work_search[freeform_names]={tags}\")\r\n if rating is not None:\r\n query.add_field(f\"work_search[rating_ids]={rating}\")\r\n if hits is not None:\r\n query.add_field(f\"work_search[hits]={hits}\")\r\n if kudos is not None:\r\n query.add_field(f\"work_search[kudos_count]={kudos}\")\r\n if crossovers is not None:\r\n query.add_field(f\"work_search[crossover]={'T' if crossovers else 'F'}\")\r\n if bookmarks is not None:\r\n query.add_field(f\"work_search[bookmarks_count]={bookmarks}\")\r\n if excluded_tags != \"\":\r\n query.add_field(f\"work_search[excluded_tag_names]={excluded_tags}\")\r\n if comments is not None:\r\n query.add_field(f\"work_search[comments_count]={comments}\")\r\n if completion_status is not None:\r\n query.add_field(f\"work_search[complete]={'T' if completion_status else 'F'}\")\r\n if sort_column != \"\":\r\n query.add_field(f\"work_search[sort_column]={sort_column}\")\r\n if sort_direction != \"\":\r\n query.add_field(f\"work_search[sort_direction]={sort_direction}\")\r\n if revised_at != \"\":\r\n query.add_field(f\"work_search[revised_at]={revised_at}\")\r\n\r\n url = f\"https://archiveofourown.org/works/search?{query.string}\"\r\n\r\n if session is None:\r\n req = requester.request(\"get\", url)\r\n else:\r\n req = session.get(url)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n soup = BeautifulSoup(req.content, features=\"lxml\")\r\n return soup\r\n" }, { "path": "AO3/series.py", "content": "from datetime import date\nfrom functools import cached_property\n\nfrom bs4 import BeautifulSoup\n\nfrom . import threadable, utils\nfrom .common import get_work_from_banner\nfrom .requester import requester\nfrom .users import User\nfrom .works import Work\n\n\nclass Series:\n def __init__(self, seriesid, session=None, load=True):\n \"\"\"Creates a new series object\n\n Args:\n seriesid (int/str): ID of the series\n session (AO3.Session, optional): Session object. Defaults to None.\n load (bool, optional): If true, the work is loaded on initialization. Defaults to True.\n\n Raises:\n utils.InvalidIdError: Invalid series ID\n \"\"\"\n \n self.id = seriesid\n self._session = session\n self._soup = None\n if load:\n self.reload()\n \n def __eq__(self, other):\n return isinstance(other, __class__) and other.id == self.id\n \n def __repr__(self):\n try:\n return f\"\" \n except:\n return f\"\"\n \n def __getstate__(self):\n d = {}\n for attr in self.__dict__:\n if isinstance(self.__dict__[attr], BeautifulSoup):\n d[attr] = (self.__dict__[attr].encode(), True)\n else:\n d[attr] = (self.__dict__[attr], False)\n return d\n \n def __setstate__(self, d):\n for attr in d:\n value, issoup = d[attr]\n if issoup:\n self.__dict__[attr] = BeautifulSoup(value, \"lxml\")\n else:\n self.__dict__[attr] = value\n \n def set_session(self, session):\n \"\"\"Sets the session used to make requests for this series\n\n Args:\n session (AO3.Session/AO3.GuestSession): session object\n \"\"\"\n \n self._session = session \n \n @threadable.threadable\n def reload(self):\n \"\"\"\n Loads information about this series.\n This function is threadable.\n \"\"\"\n \n for attr in self.__class__.__dict__:\n if isinstance(getattr(self.__class__, attr), cached_property):\n if attr in self.__dict__:\n delattr(self, attr)\n \n self._soup = self.request(f\"https://archiveofourown.org/series/{self.id}\")\n if \"Error 404\" in self._soup.text:\n raise utils.InvalidIdError(\"Cannot find series\")\n \n @threadable.threadable\n def subscribe(self):\n \"\"\"Subscribes to this series.\n This function is threadable.\n\n Raises:\n utils.AuthError: Invalid session\n \"\"\"\n \n if self._session is None or not self._session.is_authed:\n raise utils.AuthError(\"You can only subscribe to a series using an authenticated session\")\n \n utils.subscribe(self, \"Series\", self._session)\n \n @threadable.threadable\n def unsubscribe(self):\n \"\"\"Unubscribes from this series.\n This function is threadable.\n\n Raises:\n utils.AuthError: Invalid session\n \"\"\"\n \n if not self.is_subscribed:\n raise Exception(\"You are not subscribed to this series\")\n if self._session is None or not self._session.is_authed:\n raise utils.AuthError(\"You can only unsubscribe from a series using an authenticated session\")\n \n utils.subscribe(self, \"Series\", self._session, True, self._sub_id)\n \n @threadable.threadable\n def bookmark(self, notes=\"\", tags=None, collections=None, private=False, recommend=False, pseud=None):\n \"\"\"Bookmarks this series\n This function is threadable\n\n Args:\n notes (str, optional): Bookmark notes. Defaults to \"\".\n tags (list, optional): What tags to add. Defaults to None.\n collections (list, optional): What collections to add this bookmark to. Defaults to None.\n private (bool, optional): Whether this bookmark should be private. Defaults to False.\n recommend (bool, optional): Whether to recommend this bookmark. Defaults to False.\n pseud (str, optional): What pseud to add the bookmark under. Defaults to default pseud.\n\n Raises:\n utils.UnloadedError: Series isn't loaded\n utils.AuthError: Invalid session\n \"\"\"\n \n if not self.loaded:\n raise utils.UnloadedError(\"Series isn't loaded. Have you tried calling Series.reload()?\")\n \n if self._session is None:\n raise utils.AuthError(\"Invalid session\")\n \n utils.bookmark(self, self._session, notes, tags, collections, private, recommend, pseud)\n \n @threadable.threadable\n def delete_bookmark(self):\n \"\"\"Removes a bookmark from this series\n This function is threadable\n\n Raises:\n utils.UnloadedError: Series isn't loaded\n utils.AuthError: Invalid session\n \"\"\"\n \n if not self.loaded:\n raise utils.UnloadedError(\"Series isn't loaded. Have you tried calling Series.reload()?\")\n \n if self._session is None:\n raise utils.AuthError(\"Invalid session\")\n \n if self._bookmarkid is None:\n raise utils.BookmarkError(\"You don't have a bookmark here\")\n \n utils.delete_bookmark(self._bookmarkid, self._session, self.authenticity_token)\n \n @cached_property\n def _bookmarkid(self):\n form_div = self._soup.find(\"div\", {\"id\": \"bookmark-form\"})\n if form_div is None: \n return None\n if form_div.form is None:\n return None\n if \"action\" in form_div.form and form_div.form[\"action\"].startswith(\"/bookmark\"):\n text = form_div.form[\"action\"].split(\"/\")[-1]\n if text.isdigit():\n return int(text)\n return None\n return None\n \n @cached_property\n def url(self):\n \"\"\"Returns the URL to this series\n\n Returns:\n str: series URL\n \"\"\" \n\n return f\"https://archiveofourown.org/series/{self.id}\"\n \n @property\n def loaded(self):\n \"\"\"Returns True if this series has been loaded\"\"\"\n return self._soup is not None\n \n @cached_property\n def authenticity_token(self):\n \"\"\"Token used to take actions that involve this work\"\"\"\n \n if not self.loaded:\n return None\n \n token = self._soup.find(\"meta\", {\"name\": \"csrf-token\"})\n return token[\"content\"]\n \n @cached_property\n def is_subscribed(self):\n \"\"\"True if you're subscribed to this series\"\"\"\n \n if self._session is None or not self._session.is_authed:\n raise utils.AuthError(\"You can only get a series ID using an authenticated session\")\n \n form = self._soup.find(\"form\", {\"data-create-value\": \"Subscribe\"})\n input_ = form.find(\"input\", {\"name\": \"commit\", \"value\": \"Unsubscribe\"})\n return input_ is not None\n \n @cached_property\n def _sub_id(self):\n \"\"\"Returns the subscription ID. Used for unsubscribing\"\"\"\n \n if not self.is_subscribed:\n raise Exception(\"You are not subscribed to this series\")\n \n form = self._soup.find(\"form\", {\"data-create-value\": \"Subscribe\"})\n id_ = form.attrs[\"action\"].split(\"/\")[-1]\n return int(id_)\n \n @cached_property\n def name(self):\n div = self._soup.find(\"div\", {\"class\": \"series-show region\"})\n return div.h2.getText().replace(\"\\t\", \"\").replace(\"\\n\", \"\")\n \n @cached_property\n def creators(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n return [User(author.getText(), load=False) for author in dl.findAll(\"a\", {\"rel\": \"author\"})]\n \n @cached_property\n def series_begun(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n info = dl.findAll((\"dd\", \"dt\"))\n last_dt = None\n for field in info:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Series Begun:\":\n date_str = field.getText().strip()\n break\n return date(*list(map(int, date_str.split(\"-\"))))\n \n @cached_property\n def series_updated(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n info = dl.findAll((\"dd\", \"dt\"))\n last_dt = None\n for field in info:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Series Updated:\":\n date_str = field.getText().strip()\n break\n return date(*list(map(int, date_str.split(\"-\"))))\n \n @cached_property\n def words(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n stats = dl.find(\"dl\", {\"class\": \"stats\"}).findAll((\"dd\", \"dt\"))\n last_dt = None\n for field in stats:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Words:\":\n words = field.getText().strip()\n break\n return int(words.replace(\",\", \"\"))\n \n @cached_property\n def nworks(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n stats = dl.find(\"dl\", {\"class\": \"stats\"}).findAll((\"dd\", \"dt\"))\n last_dt = None\n for field in stats:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Works:\":\n works = field.getText().strip()\n break\n return int(works.replace(\",\", \"\"))\n \n @cached_property\n def complete(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n stats = dl.find(\"dl\", {\"class\": \"stats\"}).findAll((\"dd\", \"dt\"))\n last_dt = None\n for field in stats:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Complete:\":\n complete = field.getText().strip()\n break\n return True if complete == \"Yes\" else False\n \n @cached_property\n def description(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n info = dl.findAll((\"dd\", \"dt\"))\n last_dt = None\n desc = \"\"\n for field in info:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Description:\":\n desc = field.getText().strip()\n break\n return desc\n \n @cached_property\n def notes(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n info = dl.findAll((\"dd\", \"dt\"))\n last_dt = None\n notes = \"\"\n for field in info:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Notes:\":\n notes = field.getText().strip()\n break\n return notes\n \n @cached_property\n def nbookmarks(self):\n dl = self._soup.find(\"dl\", {\"class\": \"series meta group\"})\n stats = dl.find(\"dl\", {\"class\": \"stats\"}).findAll((\"dd\", \"dt\"))\n last_dt = None\n book = \"0\"\n for field in stats:\n if field.name == \"dt\":\n last_dt = field.getText().strip()\n elif last_dt == \"Bookmarks:\":\n book = field.getText().strip()\n break\n return int(book.replace(\",\", \"\")) \n \n @cached_property\n def work_list(self):\n ul = self._soup.find(\"ul\", {\"class\": \"series work index group\"})\n works = []\n for work in ul.find_all(\"li\", {\"role\": \"article\"}):\n if work.h4 is None:\n continue\n works.append(get_work_from_banner(work))\n # authors = []\n # if work.h4 is None:\n # continue\n # for a in work.h4.find_all(\"a\"):\n # if \"rel\" in a.attrs.keys():\n # if \"author\" in a[\"rel\"]:\n # authors.append(User(a.string, load=False))\n # elif a.attrs[\"href\"].startswith(\"/works\"):\n # workname = a.string\n # workid = utils.workid_from_url(a[\"href\"])\n # new = Work(workid, load=False)\n # setattr(new, \"title\", workname)\n # setattr(new, \"authors\", authors)\n # works.append(new)\n return works\n \n def get(self, *args, **kwargs):\n \"\"\"Request a web page and return a Response object\"\"\" \n \n if self._session is None:\n req = requester.request(\"get\", *args, **kwargs)\n else:\n req = requester.request(\"get\", *args, **kwargs, session=self._session.session)\n if req.status_code == 429:\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\n return req\n\n def request(self, url):\n \"\"\"Request a web page and return a BeautifulSoup object.\n\n Args:\n url (str): Url to request\n\n Returns:\n bs4.BeautifulSoup: BeautifulSoup object representing the requested page's html\n \"\"\"\n\n req = self.get(url)\n soup = BeautifulSoup(req.content, \"lxml\")\n return soup\n" }, { "path": "AO3/session.py", "content": "import datetime\r\nimport re\r\nimport time\r\nfrom functools import cached_property\r\n\r\nimport requests\r\nfrom bs4 import BeautifulSoup\r\n\r\nfrom . import threadable, utils\r\nfrom .requester import requester\r\nfrom .series import Series\r\nfrom .users import User\r\nfrom .works import Work\r\n\r\n\r\nclass GuestSession:\r\n \"\"\"\r\n AO3 guest session object\r\n \"\"\"\r\n\r\n def __init__(self):\r\n self.is_authed = False\r\n self.authenticity_token = None\r\n self.username = \"\"\r\n self.session = requests.Session()\r\n \r\n @property\r\n def user(self):\r\n return User(self.username, self, False)\r\n \r\n @threadable.threadable\r\n def comment(self, commentable, comment_text, oneshot=False, commentid=None):\r\n \"\"\"Leaves a comment on a specific work.\r\n This function is threadable.\r\n\r\n Args:\r\n commentable (Work/Chapter): Commentable object\r\n comment_text (str): Comment text (must have between 1 and 10000 characters)\r\n oneshot (bool): Should be True if the work has only one chapter. In this case, chapterid becomes workid\r\n commentid (str/int): If specified, the comment is posted as a reply to this one. Defaults to None.\r\n\r\n Raises:\r\n utils.InvalidIdError: Invalid ID\r\n utils.UnexpectedResponseError: Unknown error\r\n utils.PseudoError: Couldn't find a valid pseudonym to post under\r\n utils.DuplicateCommentError: The comment you're trying to post was already posted\r\n ValueError: Invalid name/email\r\n\r\n Returns:\r\n requests.models.Response: Response object\r\n \"\"\"\r\n \r\n response = utils.comment(commentable, comment_text, self, oneshot, commentid)\r\n return response\r\n\r\n \r\n @threadable.threadable\r\n def kudos(self, work):\r\n \"\"\"Leave a 'kudos' in a specific work.\r\n This function is threadable.\r\n\r\n Args:\r\n work (Work): ID of the work\r\n\r\n Raises:\r\n utils.UnexpectedResponseError: Unexpected response received\r\n utils.InvalidIdError: Invalid ID (work doesn't exist)\r\n\r\n Returns:\r\n bool: True if successful, False if you already left kudos there\r\n \"\"\"\r\n \r\n return utils.kudos(work, self)\r\n \r\n @threadable.threadable\r\n def refresh_auth_token(self):\r\n \"\"\"Refreshes the authenticity token.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.UnexpectedResponseError: Couldn't refresh the token\r\n \"\"\"\r\n \r\n # For some reason, the auth token in the root path only works if you're \r\n # unauthenticated. To get around that, we check if this is an authed\r\n # session and, if so, get the token from the profile page.\r\n \r\n if self.is_authed:\r\n req = self.session.get(f\"https://archiveofourown.org/users/{self.username}\")\r\n else:\r\n req = self.session.get(\"https://archiveofourown.org\")\r\n \r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n \r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n token = soup.find(\"input\", {\"name\": \"authenticity_token\"})\r\n if token is None:\r\n raise utils.UnexpectedResponseError(\"Couldn't refresh token\")\r\n self.authenticity_token = token.attrs[\"value\"]\r\n \r\n def get(self, *args, **kwargs):\r\n \"\"\"Request a web page and return a Response object\"\"\" \r\n \r\n if self.session is None:\r\n req = requester.request(\"get\", *args, **kwargs)\r\n else:\r\n req = requester.request(\"get\", *args, **kwargs, session=self.session)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n return req\r\n\r\n def request(self, url):\r\n \"\"\"Request a web page and return a BeautifulSoup object.\r\n\r\n Args:\r\n url (str): Url to request\r\n\r\n Returns:\r\n bs4.BeautifulSoup: BeautifulSoup object representing the requested page's html\r\n \"\"\"\r\n\r\n req = self.get(url)\r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n return soup\r\n\r\n def post(self, *args, **kwargs):\r\n \"\"\"Make a post request with the current session\r\n\r\n Returns:\r\n requests.Request\r\n \"\"\"\r\n\r\n req = self.session.post(*args, **kwargs)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n return req\r\n \r\n def __del__(self):\r\n self.session.close()\r\n\r\nclass Session(GuestSession):\r\n \"\"\"\r\n AO3 session object\r\n \"\"\"\r\n\r\n def __init__(self, username, password):\r\n \"\"\"Creates a new AO3 session object\r\n\r\n Args:\r\n username (str): AO3 username\r\n password (str): AO3 password\r\n\r\n Raises:\r\n utils.LoginError: Login was unsucessful (wrong username or password)\r\n \"\"\"\r\n\r\n super().__init__()\r\n self.is_authed = True\r\n self.username = username\r\n self.url = \"https://archiveofourown.org/users/%s\"%self.username\r\n \r\n self.session = requests.Session()\r\n \r\n soup = self.request(\"https://archiveofourown.org/users/login\")\r\n self.authenticity_token = soup.find(\"input\", {\"name\": 'authenticity_token'})[\"value\"]\r\n payload = {'user[login]': username,\r\n 'user[password]': password,\r\n 'authenticity_token': self.authenticity_token}\r\n post = self.post(\"https://archiveofourown.org/users/login\", params=payload, allow_redirects=False)\r\n if not post.status_code == 302:\r\n raise utils.LoginError(\"Invalid username or password\")\r\n\r\n self._subscriptions_url = \"https://archiveofourown.org/users/{0}/subscriptions?page={1:d}\"\r\n self._bookmarks_url = \"https://archiveofourown.org/users/{0}/bookmarks?page={1:d}\"\r\n self._history_url = \"https://archiveofourown.org/users/{0}/readings?page={1:d}\"\r\n \r\n self._bookmarks = None\r\n self._subscriptions = None\r\n self._history = None\r\n \r\n def __getstate__(self):\r\n d = {}\r\n for attr in self.__dict__:\r\n if isinstance(self.__dict__[attr], BeautifulSoup):\r\n d[attr] = (self.__dict__[attr].encode(), True)\r\n else:\r\n d[attr] = (self.__dict__[attr], False)\r\n return d\r\n \r\n def __setstate__(self, d):\r\n for attr in d:\r\n value, issoup = d[attr]\r\n if issoup:\r\n self.__dict__[attr] = BeautifulSoup(value, \"lxml\")\r\n else:\r\n self.__dict__[attr] = value\r\n \r\n def clear_cache(self):\r\n for attr in self.__class__.__dict__:\r\n if isinstance(getattr(self.__class__, attr), cached_property):\r\n if attr in self.__dict__:\r\n delattr(self, attr)\r\n self._bookmarks = None\r\n self._subscriptions = None\r\n \r\n @cached_property\r\n def _subscription_pages(self):\r\n url = self._subscriptions_url.format(self.username, 1)\r\n soup = self.request(url)\r\n pages = soup.find(\"ol\", {\"title\": \"pagination\"})\r\n if pages is None:\r\n return 1\r\n n = 1\r\n for li in pages.findAll(\"li\"):\r\n text = li.getText()\r\n if text.isdigit():\r\n n = int(text)\r\n return n\r\n \r\n def get_work_subscriptions(self, use_threading=False):\r\n \"\"\"\r\n Get subscribed works. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of work subscriptions\r\n \"\"\"\r\n \r\n subs = self.get_subscriptions(use_threading)\r\n return list(filter(lambda obj: isinstance(obj, Work), subs))\r\n \r\n def get_series_subscriptions(self, use_threading=False):\r\n \"\"\"\r\n Get subscribed series. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of series subscriptions\r\n \"\"\"\r\n \r\n subs = self.get_subscriptions(use_threading)\r\n return list(filter(lambda obj: isinstance(obj, Series), subs))\r\n \r\n def get_user_subscriptions(self, use_threading=False):\r\n \"\"\"\r\n Get subscribed users. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of users subscriptions\r\n \"\"\"\r\n \r\n subs = self.get_subscriptions(use_threading)\r\n return list(filter(lambda obj: isinstance(obj, User), subs))\r\n \r\n def get_subscriptions(self, use_threading=False):\r\n \"\"\"\r\n Get user's subscriptions. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of subscriptions\r\n \"\"\"\r\n \r\n if self._subscriptions is None:\r\n if use_threading:\r\n self.load_subscriptions_threaded()\r\n else:\r\n self._subscriptions = []\r\n for page in range(self._subscription_pages):\r\n self._load_subscriptions(page=page+1)\r\n return self._subscriptions\r\n \r\n @threadable.threadable\r\n def load_subscriptions_threaded(self):\r\n \"\"\"\r\n Get subscribed works using threads.\r\n This function is threadable.\r\n \"\"\" \r\n \r\n threads = []\r\n self._subscriptions = []\r\n for page in range(self._subscription_pages):\r\n threads.append(self._load_subscriptions(page=page+1, threaded=True))\r\n for thread in threads:\r\n thread.join()\r\n\r\n @threadable.threadable\r\n def _load_subscriptions(self, page=1): \r\n url = self._subscriptions_url.format(self.username, page)\r\n soup = self.request(url)\r\n subscriptions = soup.find(\"dl\", {\"class\": \"subscription index group\"})\r\n for sub in subscriptions.find_all(\"dt\"):\r\n type_ = \"work\"\r\n user = None\r\n series = None\r\n workid = None\r\n workname = None\r\n authors = []\r\n for a in sub.find_all(\"a\"):\r\n if \"rel\" in a.attrs.keys():\r\n if \"author\" in a[\"rel\"]:\r\n authors.append(User(str(a.string), load=False))\r\n elif a[\"href\"].startswith(\"/works\"):\r\n workname = str(a.string)\r\n workid = utils.workid_from_url(a[\"href\"])\r\n elif a[\"href\"].startswith(\"/users\"):\r\n type_ = \"user\"\r\n user = User(str(a.string), load=False)\r\n else:\r\n type_ = \"series\"\r\n workname = str(a.string)\r\n series = int(a[\"href\"].split(\"/\")[-1])\r\n if type_ == \"work\":\r\n new = Work(workid, load=False)\r\n setattr(new, \"title\", workname)\r\n setattr(new, \"authors\", authors)\r\n self._subscriptions.append(new)\r\n elif type_ == \"user\":\r\n self._subscriptions.append(user)\r\n elif type_ == \"series\":\r\n new = Series(series, load=False)\r\n setattr(new, \"name\", workname)\r\n setattr(new, \"authors\", authors)\r\n self._subscriptions.append(new)\r\n\r\n @cached_property\r\n def _history_pages(self):\r\n url = self._history_url.format(self.username, 1)\r\n soup = self.request(url)\r\n pages = soup.find(\"ol\", {\"title\": \"pagination\"})\r\n if pages is None:\r\n return 1\r\n n = 1\r\n for li in pages.findAll(\"li\"):\r\n text = li.getText()\r\n if text.isdigit():\r\n n = int(text)\r\n return n\r\n\r\n def get_history(self, hist_sleep=3, start_page=0, max_pages=None, timeout_sleep=60):\r\n \"\"\"\r\n Get history works. Loads them if they haven't been previously.\r\n\r\n Arguments:\r\n hist_sleep (int to sleep between requests)\r\n start_page (int for page to start on, zero-indexed)\r\n max_pages (int for page to end on, zero-indexed)\r\n timeout_sleep (int, if set will attempt to recovery from http errors, likely timeouts, if set to None will just attempt to load)\r\n\r\n takes two arguments the first hist_sleep is an int and is a sleep to run between pages of history to load to avoid hitting the rate limiter, the second is an int of the maximum number of pages of history to load, by default this is None so loads them all.\r\n\r\n Returns:\r\n list: List of tuples (Work, number-of-visits, datetime-last-visited)\r\n \"\"\"\r\n \r\n if self._history is None:\r\n self._history = []\r\n for page in range(start_page, self._history_pages):\r\n # If we are attempting to recover from errors then\r\n # catch and loop, otherwise just call and go\r\n if timeout_sleep is None:\r\n self._load_history(page=page+1)\r\n \r\n else:\r\n loaded=False\r\n while loaded == False:\r\n try:\r\n self._load_history(page=page+1)\r\n # print(f\"Read history page {page+1}\")\r\n loaded = True\r\n\r\n except utils.HTTPError:\r\n # print(f\"History being rate limited, sleeping for {timeout_sleep} seconds\")\r\n time.sleep(timeout_sleep)\r\n\r\n # Check for maximum history page load\r\n if max_pages is not None and page >= max_pages:\r\n return self._history\r\n\r\n # Again attempt to avoid rate limiter, sleep for a few\r\n # seconds between page requests.\r\n if hist_sleep is not None and hist_sleep > 0:\r\n time.sleep(hist_sleep)\r\n\r\n return self._history\r\n\r\n def _load_history(self, page=1): \r\n url = self._history_url.format(self.username, page)\r\n soup = self.request(url)\r\n history = soup.find(\"ol\", {\"class\": \"reading work index group\"})\r\n for item in history.find_all(\"li\", {\"role\": \"article\"}):\r\n # authors = []\r\n workname = None\r\n workid = None\r\n for a in item.h4.find_all(\"a\"):\r\n if a.attrs[\"href\"].startswith(\"/works\"):\r\n workname = str(a.string)\r\n workid = utils.workid_from_url(a[\"href\"])\r\n\r\n visited_date = None\r\n visited_num = 1\r\n for viewed in item.find_all(\"h4\", {\"class\": \"viewed heading\" }):\r\n data_string = str(viewed)\r\n date_str = re.search('Last visited: (\\d{2} .+ \\d{4})', data_string)\r\n if date_str is not None:\r\n raw_date = date_str.group(1)\r\n date_time_obj = datetime.datetime.strptime(date_str.group(1), '%d %b %Y')\r\n visited_date = date_time_obj\r\n \r\n visited_str = re.search('Visited (\\d+) times', data_string)\r\n if visited_str is not None:\r\n visited_num = int(visited_str.group(1))\r\n \r\n\r\n if workname != None and workid != None:\r\n new = Work(workid, load=False)\r\n setattr(new, \"title\", workname)\r\n # setattr(new, \"authors\", authors)\r\n hist_item = [ new, visited_num, visited_date ]\r\n # print(hist_item)\r\n if new not in self._history:\r\n self._history.append(hist_item)\r\n \r\n @cached_property\r\n def _bookmark_pages(self):\r\n url = self._bookmarks_url.format(self.username, 1)\r\n soup = self.request(url)\r\n pages = soup.find(\"ol\", {\"title\": \"pagination\"})\r\n if pages is None:\r\n return 1\r\n n = 1\r\n for li in pages.findAll(\"li\"):\r\n text = li.getText()\r\n if text.isdigit():\r\n n = int(text)\r\n return n\r\n \r\n def get_bookmarks(self, use_threading=False):\r\n \"\"\"\r\n Get bookmarked works. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of tuples (workid, workname, authors)\r\n \"\"\"\r\n \r\n if self._bookmarks is None:\r\n if use_threading:\r\n self.load_bookmarks_threaded()\r\n else:\r\n self._bookmarks = []\r\n for page in range(self._bookmark_pages):\r\n self._load_bookmarks(page=page+1)\r\n return self._bookmarks\r\n \r\n @threadable.threadable\r\n def load_bookmarks_threaded(self):\r\n \"\"\"\r\n Get bookmarked works using threads.\r\n This function is threadable.\r\n \"\"\" \r\n \r\n threads = []\r\n self._bookmarks = []\r\n for page in range(self._bookmark_pages):\r\n threads.append(self._load_bookmarks(page=page+1, threaded=True))\r\n for thread in threads:\r\n thread.join()\r\n \r\n @threadable.threadable\r\n def _load_bookmarks(self, page=1): \r\n url = self._bookmarks_url.format(self.username, page)\r\n soup = self.request(url)\r\n bookmarks = soup.find(\"ol\", {\"class\": \"bookmark index group\"})\r\n for bookm in bookmarks.find_all(\"li\", {\"class\": [\"bookmark\", \"index\", \"group\"]}):\r\n authors = []\r\n recommended = False\r\n workid = -1\r\n if bookm.h4 is not None:\r\n for a in bookm.h4.find_all(\"a\"):\r\n if \"rel\" in a.attrs.keys():\r\n if \"author\" in a[\"rel\"]:\r\n authors.append(User(str(a.string), load=False))\r\n elif a.attrs[\"href\"].startswith(\"/works\"):\r\n workname = str(a.string)\r\n workid = utils.workid_from_url(a[\"href\"])\r\n\r\n # Get whether the bookmark is recommended\r\n for span in bookm.p.find_all(\"span\"):\r\n if \"title\" in span.attrs.keys():\r\n if span[\"title\"] == \"Rec\":\r\n recommended = True\r\n\r\n \r\n if workid != -1:\r\n new = Work(workid, load=False)\r\n setattr(new, \"title\", workname)\r\n setattr(new, \"authors\", authors)\r\n setattr(new, \"recommended\", recommended)\r\n if new not in self._bookmarks:\r\n self._bookmarks.append(new)\r\n \r\n @cached_property\r\n def bookmarks(self):\r\n \"\"\"Get the number of your bookmarks.\r\n Must be logged in to use.\r\n\r\n Returns:\r\n int: Number of bookmarks\r\n \"\"\"\r\n\r\n url = self._bookmarks_url.format(self.username, 1)\r\n soup = self.request(url)\r\n div = soup.find(\"div\", {\"class\": \"bookmarks-index dashboard filtered region\"})\r\n h2 = div.h2.text.split()\r\n return int(h2[4].replace(',', ''))\r\n \r\n def get_statistics(self, year=None):\r\n year = \"All+Years\" if year is None else str(year)\r\n url = f\"https://archiveofourown.org/users/{self.username}/stats?year={year}\"\r\n soup = self.request(url) \r\n stats = {}\r\n dt = soup.find(\"dl\", {\"class\": \"statistics meta group\"})\r\n if dt is not None:\r\n for field in dt.findAll(\"dt\"):\r\n name = field.getText()[:-1].lower().replace(\" \", \"_\")\r\n if field.next_sibling is not None and field.next_sibling.next_sibling is not None:\r\n value = field.next_sibling.next_sibling.getText().replace(\",\", \"\")\r\n if value.isdigit():\r\n stats[name] = int(value)\r\n \r\n return stats\r\n\r\n @staticmethod\r\n def str_format(string):\r\n \"\"\"Formats a given string\r\n\r\n Args:\r\n string (str): String to format\r\n\r\n Returns:\r\n str: Formatted string\r\n \"\"\"\r\n\r\n return string.replace(\",\", \"\")\r\n\r\n def get_marked_for_later(self, sleep=1, timeout_sleep=60):\r\n \"\"\"\r\n Gets every marked for later work\r\n\r\n Arguments:\r\n sleep (int): The time to wait between page requests\r\n timeout_sleep (int): The time to wait after the rate limit is hit\r\n\r\n Returns:\r\n works (list): All marked for later works\r\n \"\"\"\r\n pageRaw = self.request(f\"https://archiveofourown.org/users/{self.username}/readings?page=1&show=to-read\").find(\"ol\", {\"class\": \"pagination actions\"}).find_all(\"li\")\r\n maxPage = int(pageRaw[len(pageRaw)-2].text)\r\n works = []\r\n for page in range(maxPage):\r\n grabbed = False\r\n while grabbed == False:\r\n try:\r\n workPage = self.request(f\"https://archiveofourown.org/users/{self.username}/readings?page={page+1}&show=to-read\")\r\n worksRaw = workPage.find_all(\"li\", {\"role\": \"article\"})\r\n for work in worksRaw:\r\n try:\r\n workId = int(work.h4.a.get(\"href\").split(\"/\")[2])\r\n works.append(Work(workId, session=self, load=False))\r\n except AttributeError:\r\n pass\r\n grabbed = True\r\n except utils.HTTPError:\r\n time.sleep(timeout_sleep)\r\n time.sleep(sleep)\r\n return works\r\n" }, { "path": "AO3/threadable.py", "content": "import threading\n\n\ndef threadable(func):\n \"\"\"Allows the function to be ran as a thread using the 'threaded' argument\"\"\"\n \n def new(*args, threaded=False, **kwargs):\n if threaded:\n thread = threading.Thread(target=func, args=args, kwargs=kwargs)\n thread.start()\n return thread\n else:\n return func(*args, **kwargs)\n \n new.__doc__ = func.__doc__\n new.__name__ = func.__name__\n new._threadable = True\n return new\n \nclass ThreadPool:\n def __init__(self, maximum=None):\n self.maximum = maximum\n self._tasks = []\n self._threads = []\n \n def add_task(self, task):\n self._tasks.append(task)\n \n @threadable\n def start(self):\n while len(self._threads) != 0 or len(self._tasks) != 0:\n self._threads[:] = filter(lambda thread: thread.is_alive(), self._threads)\n for _ in range(min(self.maximum-len(self._threads), len(self._tasks))):\n self._threads.append(self._tasks.pop(0)(threaded=True))\n" }, { "path": "AO3/users.py", "content": "import datetime\r\nfrom functools import cached_property\r\n\r\nimport requests\r\nfrom bs4 import BeautifulSoup\r\n\r\nfrom . import threadable, utils\r\nfrom .common import get_work_from_banner\r\nfrom .requester import requester\r\n\r\n\r\nclass User:\r\n \"\"\"\r\n AO3 user object\r\n \"\"\"\r\n\r\n def __init__(self, username, session=None, load=True):\r\n \"\"\"Creates a new AO3 user object\r\n\r\n Args:\r\n username (str): AO3 username\r\n session (AO3.Session, optional): Used to access additional info\r\n load (bool, optional): If true, the user is loaded on initialization. Defaults to True.\r\n \"\"\"\r\n\r\n self.username = username\r\n self._session = session\r\n self._soup_works = None\r\n self._soup_profile = None\r\n self._soup_bookmarks = None\r\n self._works = None\r\n self._bookmarks = None\r\n if load:\r\n self.reload()\r\n \r\n def __repr__(self):\r\n return f\"\"\r\n \r\n def __eq__(self, other):\r\n return isinstance(other, __class__) and other.username == self.username\r\n \r\n def __getstate__(self):\r\n d = {}\r\n for attr in self.__dict__:\r\n if isinstance(self.__dict__[attr], BeautifulSoup):\r\n d[attr] = (self.__dict__[attr].encode(), True)\r\n else:\r\n d[attr] = (self.__dict__[attr], False)\r\n return d\r\n \r\n def __setstate__(self, d):\r\n for attr in d:\r\n value, issoup = d[attr]\r\n if issoup:\r\n self.__dict__[attr] = BeautifulSoup(value, \"lxml\")\r\n else:\r\n self.__dict__[attr] = value\r\n \r\n def set_session(self, session):\r\n \"\"\"Sets the session used to make requests for this work\r\n\r\n Args:\r\n session (AO3.Session/AO3.GuestSession): session object\r\n \"\"\"\r\n \r\n self._session = session \r\n \r\n @threadable.threadable\r\n def reload(self):\r\n \"\"\"\r\n Loads information about this user.\r\n This function is threadable.\r\n \"\"\"\r\n \r\n for attr in self.__class__.__dict__:\r\n if isinstance(getattr(self.__class__, attr), cached_property):\r\n if attr in self.__dict__:\r\n delattr(self, attr)\r\n \r\n @threadable.threadable\r\n def req_works(username):\r\n self._soup_works = self.request(f\"https://archiveofourown.org/users/{username}/works\")\r\n token = self._soup_works.find(\"meta\", {\"name\": \"csrf-token\"})\r\n setattr(self, \"authenticity_token\", token[\"content\"])\r\n \r\n @threadable.threadable\r\n def req_profile(username): \r\n self._soup_profile = self.request(f\"https://archiveofourown.org/users/{username}/profile\")\r\n token = self._soup_profile.find(\"meta\", {\"name\": \"csrf-token\"})\r\n setattr(self, \"authenticity_token\", token[\"content\"])\r\n\r\n @threadable.threadable\r\n def req_bookmarks(username): \r\n self._soup_bookmarks = self.request(f\"https://archiveofourown.org/users/{username}/bookmarks\")\r\n token = self._soup_bookmarks.find(\"meta\", {\"name\": \"csrf-token\"})\r\n setattr(self, \"authenticity_token\", token[\"content\"])\r\n \r\n rs = [req_works(self.username, threaded=True),\r\n req_profile(self.username, threaded=True),\r\n req_bookmarks(self.username, threaded=True)]\r\n for r in rs:\r\n r.join()\r\n\r\n self._works = None\r\n self._bookmarks = None\r\n \r\n def get_avatar(self):\r\n \"\"\"Returns a tuple containing the name of the file and its data\r\n\r\n Returns:\r\n tuple: (name: str, img: bytes)\r\n \"\"\"\r\n \r\n icon = self._soup_profile.find(\"p\", {\"class\": \"icon\"})\r\n src = icon.img.attrs[\"src\"]\r\n name = src.split(\"/\")[-1].split(\"?\")[0]\r\n img = self.get(src).content\r\n return name, img\r\n \r\n @threadable.threadable\r\n def subscribe(self):\r\n \"\"\"Subscribes to this user.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only subscribe to a user using an authenticated session\")\r\n \r\n utils.subscribe(self, \"User\", self._session)\r\n \r\n @threadable.threadable\r\n def unsubscribe(self):\r\n \"\"\"Unubscribes from this user.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if not self.is_subscribed:\r\n raise Exception(\"You are not subscribed to this user\")\r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only unsubscribe from a user using an authenticated session\")\r\n \r\n utils.subscribe(self, \"User\", self._session, True, self._sub_id)\r\n \r\n @property\r\n def id(self):\r\n id_ = self._soup_profile.find(\"input\", {\"id\": \"subscription_subscribable_id\"})\r\n return int(id_[\"value\"]) if id_ is not None else None\r\n \r\n @cached_property\r\n def is_subscribed(self):\r\n \"\"\"True if you're subscribed to this user\"\"\"\r\n \r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only get a user ID using an authenticated session\")\r\n \r\n header = self._soup_profile.find(\"div\", {\"class\": \"primary header module\"})\r\n input_ = header.find(\"input\", {\"name\": \"commit\", \"value\": \"Unsubscribe\"})\r\n return input_ is not None\r\n \r\n @property\r\n def loaded(self):\r\n \"\"\"Returns True if this user has been loaded\"\"\"\r\n return self._soup_profile is not None\r\n \r\n # @cached_property\r\n # def authenticity_token(self):\r\n # \"\"\"Token used to take actions that involve this user\"\"\"\r\n \r\n # if not self.loaded:\r\n # return None\r\n \r\n # token = self._soup_profile.find(\"meta\", {\"name\": \"csrf-token\"})\r\n # return token[\"content\"]\r\n \r\n @cached_property\r\n def user_id(self):\r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only get a user ID using an authenticated session\")\r\n \r\n header = self._soup_profile.find(\"div\", {\"class\": \"primary header module\"})\r\n input_ = header.find(\"input\", {\"name\": \"subscription[subscribable_id]\"})\r\n if input_ is None:\r\n raise utils.UnexpectedResponseError(\"Couldn't fetch user ID\")\r\n return int(input_.attrs[\"value\"])\r\n \r\n @cached_property\r\n def _sub_id(self):\r\n \"\"\"Returns the subscription ID. Used for unsubscribing\"\"\"\r\n \r\n if not self.is_subscribed:\r\n raise Exception(\"You are not subscribed to this user\")\r\n \r\n header = self._soup_profile.find(\"div\", {\"class\": \"primary header module\"})\r\n id_ = header.form.attrs[\"action\"].split(\"/\")[-1]\r\n return int(id_)\r\n\r\n @cached_property\r\n def works(self):\r\n \"\"\"Returns the number of works authored by this user\r\n\r\n Returns:\r\n int: Number of works\r\n \"\"\"\r\n\r\n div = self._soup_works.find(\"div\", {\"class\": \"works-index dashboard filtered region\"})\r\n h2 = div.h2.text.split()\r\n return int(h2[4].replace(',', '')) \r\n\r\n @cached_property\r\n def _works_pages(self):\r\n pages = self._soup_works.find(\"ol\", {\"title\": \"pagination\"})\r\n if pages is None:\r\n return 1\r\n n = 1\r\n for li in pages.findAll(\"li\"):\r\n text = li.getText()\r\n if text.isdigit():\r\n n = int(text)\r\n return n\r\n \r\n def get_works(self, use_threading=False):\r\n \"\"\"\r\n Get works authored by this user.\r\n\r\n Returns:\r\n list: List of works\r\n \"\"\"\r\n \r\n if self._works is None:\r\n if use_threading:\r\n self.load_works_threaded()\r\n else:\r\n self._works = []\r\n for page in range(self._works_pages):\r\n self._load_works(page=page+1)\r\n return self._works\r\n \r\n @threadable.threadable\r\n def load_works_threaded(self):\r\n \"\"\"\r\n Get the user's works using threads.\r\n This function is threadable.\r\n \"\"\" \r\n \r\n threads = []\r\n self._works = []\r\n for page in range(self._works_pages):\r\n threads.append(self._load_works(page=page+1, threaded=True))\r\n for thread in threads:\r\n thread.join()\r\n\r\n @threadable.threadable\r\n def _load_works(self, page=1):\r\n from .works import Work\r\n self._soup_works = self.request(f\"https://archiveofourown.org/users/{self.username}/works?page={page}\")\r\n \r\n ol = self._soup_works.find(\"ol\", {\"class\": \"work index group\"})\r\n\r\n for work in ol.find_all(\"li\", {\"role\": \"article\"}):\r\n if work.h4 is None:\r\n continue\r\n self._works.append(get_work_from_banner(work))\r\n\r\n @cached_property\r\n def bookmarks(self):\r\n \"\"\"Returns the number of works user has bookmarked\r\n\r\n Returns:\r\n int: Number of bookmarks \r\n \"\"\"\r\n\r\n div = self._soup_bookmarks.find(\"div\", {\"class\": \"bookmarks-index dashboard filtered region\"})\r\n h2 = div.h2.text.split()\r\n return int(h2[4].replace(',', '')) \r\n\r\n @cached_property\r\n def _bookmarks_pages(self):\r\n pages = self._soup_bookmarks.find(\"ol\", {\"title\": \"pagination\"})\r\n if pages is None:\r\n return 1\r\n n = 1\r\n for li in pages.findAll(\"li\"):\r\n text = li.getText()\r\n if text.isdigit():\r\n n = int(text)\r\n return n\r\n\r\n def get_bookmarks(self, use_threading=False):\r\n \"\"\"\r\n Get this user's bookmarked works. Loads them if they haven't been previously\r\n\r\n Returns:\r\n list: List of works\r\n \"\"\"\r\n \r\n if self._bookmarks is None:\r\n if use_threading:\r\n self.load_bookmarks_threaded()\r\n else:\r\n self._bookmarks = []\r\n for page in range(self._bookmarks_pages):\r\n self._load_bookmarks(page=page+1)\r\n return self._bookmarks\r\n \r\n @threadable.threadable\r\n def load_bookmarks_threaded(self):\r\n \"\"\"\r\n Get the user's bookmarks using threads.\r\n This function is threadable.\r\n \"\"\" \r\n \r\n threads = []\r\n self._bookmarks = []\r\n for page in range(self._bookmarks_pages):\r\n threads.append(self._load_bookmarks(page=page+1, threaded=True))\r\n for thread in threads:\r\n thread.join()\r\n\r\n @threadable.threadable\r\n def _load_bookmarks(self, page=1):\r\n from .works import Work\r\n self._soup_bookmarks = self.request(f\"https://archiveofourown.org/users/{self.username}/bookmarks?page={page}\")\r\n \r\n ol = self._soup_bookmarks.find(\"ol\", {\"class\": \"bookmark index group\"})\r\n\r\n for work in ol.find_all(\"li\", {\"role\": \"article\"}):\r\n authors = []\r\n if work.h4 is None:\r\n continue\r\n self._bookmarks.append(get_work_from_banner(work))\r\n \r\n @cached_property\r\n def bio(self):\r\n \"\"\"Returns the user's bio\r\n\r\n Returns:\r\n str: User's bio\r\n \"\"\"\r\n\r\n div = self._soup_profile.find(\"div\", {\"class\": \"bio module\"})\r\n if div is None:\r\n return \"\"\r\n blockquote = div.find(\"blockquote\", {\"class\": \"userstuff\"})\r\n return blockquote.getText() if blockquote is not None else \"\" \r\n \r\n @cached_property\r\n def url(self):\r\n \"\"\"Returns the URL to the user's profile\r\n\r\n Returns:\r\n str: user profile URL\r\n \"\"\"\r\n\r\n return \"https://archiveofourown.org/users/%s\"%self.username \r\n\r\n def get(self, *args, **kwargs):\r\n \"\"\"Request a web page and return a Response object\"\"\" \r\n \r\n if self._session is None:\r\n req = requester.request(\"get\", *args, **kwargs)\r\n else:\r\n req = requester.request(\"get\", *args, **kwargs, session=self._session.session)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n return req\r\n\r\n def request(self, url):\r\n \"\"\"Request a web page and return a BeautifulSoup object.\r\n\r\n Args:\r\n url (str): Url to request\r\n\r\n Returns:\r\n bs4.BeautifulSoup: BeautifulSoup object representing the requested page's html\r\n \"\"\"\r\n\r\n req = self.get(url)\r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n return soup\r\n\r\n @staticmethod\r\n def str_format(string):\r\n \"\"\"Formats a given string\r\n\r\n Args:\r\n string (str): String to format\r\n\r\n Returns:\r\n str: Formatted string\r\n \"\"\"\r\n\r\n return string.replace(\",\", \"\")\r\n\r\n @property\r\n def work_pages(self):\r\n \"\"\"\r\n Returns how many pages of works a user has\r\n\r\n Returns:\r\n int: Amount of pages\r\n \"\"\"\r\n return self._works_pages\r\n" }, { "path": "AO3/utils.py", "content": "import os\r\nimport pickle\r\nimport re\r\n\r\nfrom bs4 import BeautifulSoup\r\n\r\nfrom .requester import requester\r\nfrom .common import url_join\r\n\r\n_FANDOMS = None\r\n_LANGUAGES = None\r\n\r\nAO3_AUTH_ERROR_URL = \"https://archiveofourown.org/auth_error\"\r\n\r\n\r\nclass LoginError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n\r\nclass UnloadedError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass UnexpectedResponseError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass InvalidIdError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass DownloadError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass AuthError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors \r\n \r\nclass DuplicateCommentError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass PseudError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass HTTPError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n \r\nclass BookmarkError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n\r\nclass CollectError(Exception):\r\n def __init__(self, message, errors=[]):\r\n super().__init__(message)\r\n self.errors = errors\r\n\r\nclass Query:\r\n def __init__(self):\r\n self.fields = []\r\n \r\n def add_field(self, text):\r\n self.fields.append(text)\r\n\r\n @property\r\n def string(self):\r\n return '&'.join(self.fields)\r\n\r\n\r\nclass Constraint:\r\n \"\"\"Represents a bounding box of a value\r\n \"\"\"\r\n\r\n def __init__(self, lowerbound=0, upperbound=None):\r\n \"\"\"Creates a new Constraint object\r\n\r\n Args:\r\n lowerbound (int, optional): Constraint lowerbound. Defaults to 0.\r\n upperbound (int, optional): Constraint upperbound. Defaults to None.\r\n \"\"\"\r\n \r\n self._lb = lowerbound\r\n self._ub = upperbound\r\n\r\n @property\r\n def string(self):\r\n \"\"\"Returns the string representation of this constraint\r\n\r\n Returns:\r\n str: string representation\r\n \"\"\"\r\n\r\n if self._lb == 0:\r\n return f\"<{self._ub}\"\r\n elif self._ub is None:\r\n return f\">{self._lb}\"\r\n elif self._ub == self._lb:\r\n return str(self._lb)\r\n else:\r\n return f\"{self._lb}-{self._ub}\"\r\n\r\n def __str__(self):\r\n return self.string\r\n \r\ndef word_count(text):\r\n return len(tuple(filter(lambda w: w != \"\", re.split(\" |\\n|\\t\", text))))\r\n \r\ndef set_rqtw(value):\r\n \"\"\"Sets the requests per time window parameter for the AO3 requester\"\"\"\r\n requester.setRQTW(value)\r\n \r\ndef set_timew(value):\r\n \"\"\"Sets the time window parameter for the AO3 requester\"\"\"\r\n requester.setTimeW(value)\r\n \r\ndef limit_requests(limit=True):\r\n \"\"\"Toggles request limiting\"\"\"\r\n if limit:\r\n requester.setRQTW(12)\r\n else:\r\n requester.setRQTW(-1)\r\n \r\ndef load_fandoms():\r\n \"\"\"Loads fandoms into memory\r\n\r\n Raises:\r\n FileNotFoundError: No resource was found\r\n \"\"\"\r\n \r\n global _FANDOMS\r\n \r\n fandom_path = os.path.join(os.path.dirname(__file__), \"resources\", \"fandoms\")\r\n if not os.path.isdir(fandom_path):\r\n raise FileNotFoundError(\"No fandom resources have been downloaded. Try AO3.extra.download()\")\r\n files = os.listdir(fandom_path)\r\n _FANDOMS = []\r\n for file in files:\r\n with open(os.path.join(fandom_path, file), \"rb\") as f:\r\n _FANDOMS += pickle.load(f)\r\n \r\ndef load_languages():\r\n \"\"\"Loads languages into memory\r\n\r\n Raises:\r\n FileNotFoundError: No resource was found\r\n \"\"\"\r\n \r\n global _LANGUAGES\r\n \r\n language_path = os.path.join(os.path.dirname(__file__), \"resources\", \"languages\")\r\n if not os.path.isdir(language_path):\r\n raise FileNotFoundError(\"No language resources have been downloaded. Try AO3.extra.download()\")\r\n files = os.listdir(language_path)\r\n _LANGUAGES = []\r\n for file in files:\r\n with open(os.path.join(language_path, file), \"rb\") as f:\r\n _LANGUAGES += pickle.load(f)\r\n \r\ndef get_languages():\r\n \"\"\"Returns all available languages\"\"\"\r\n return _LANGUAGES[:]\r\n\r\ndef search_fandom(fandom_string):\r\n \"\"\"Searches for a fandom that matches the given string\r\n\r\n Args:\r\n fandom_string (str): query string\r\n\r\n Raises:\r\n UnloadedError: load_fandoms() wasn't called\r\n UnloadedError: No resources were downloaded\r\n\r\n Returns:\r\n list: All results matching 'fandom_string'\r\n \"\"\"\r\n \r\n if _FANDOMS is None:\r\n raise UnloadedError(\"Did you forget to call AO3.utils.load_fandoms()?\")\r\n if _FANDOMS == []:\r\n raise UnloadedError(\"Did you forget to download the required resources with AO3.extra.download()?\")\r\n results = []\r\n for fandom in _FANDOMS:\r\n if fandom_string.lower() in fandom.lower():\r\n results.append(fandom)\r\n return results\r\n \r\ndef workid_from_url(url):\r\n \"\"\"Get the workid from an archiveofourown.org website url\r\n\r\n Args:\r\n url (str): Work URL \r\n\r\n Returns:\r\n int: Work ID\r\n \"\"\"\r\n split_url = url.split(\"/\")\r\n try:\r\n index = split_url.index(\"works\")\r\n except ValueError:\r\n return\r\n if len(split_url) >= index+1:\r\n workid = split_url[index+1].split(\"?\")[0]\r\n if workid.isdigit():\r\n return int(workid)\r\n return\r\n\r\ndef comment(commentable, comment_text, session, fullwork=False, commentid=None, email=\"\", name=\"\", pseud=None):\r\n \"\"\"Leaves a comment on a specific work\r\n\r\n Args:\r\n commentable (Work/Chapter): Chapter/Work object\r\n comment_text (str): Comment text (must have between 1 and 10000 characters)\r\n fullwork (bool): Should be True if the work has only one chapter or if the comment is to be posted on the full work.\r\n session (AO3.Session/AO3.GuestSession): Session object to request with.\r\n commentid (str/int): If specified, the comment is posted as a reply to this comment. Defaults to None.\r\n email (str): Email to post with. Only used if sess is None. Defaults to \"\".\r\n name (str): Name that will appear on the comment. Only used if sess is None. Defaults to \"\".\r\n pseud (str, optional): What pseud to add the comment under. Defaults to default pseud.\r\n\r\n Raises:\r\n utils.InvalidIdError: Invalid ID\r\n utils.UnexpectedResponseError: Unknown error\r\n utils.PseudError: Couldn't find a valid pseudonym to post under\r\n utils.DuplicateCommentError: The comment you're trying to post was already posted\r\n ValueError: Invalid name/email\r\n\r\n Returns:\r\n requests.models.Response: Response object\r\n \"\"\"\r\n\r\n if commentable.authenticity_token is not None:\r\n at = commentable.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n headers = {\r\n \"x-requested-with\": \"XMLHttpRequest\",\r\n \"x-newrelic-id\": \"VQcCWV9RGwIJVFFRAw==\",\r\n \"x-csrf-token\": at\r\n }\r\n \r\n data = {}\r\n if fullwork:\r\n data[\"work_id\"] = str(commentable.id)\r\n else:\r\n data[\"chapter_id\"] = str(commentable.id)\r\n if commentid is not None:\r\n data[\"comment_id\"] = commentid\r\n \r\n if session.is_authed:\r\n if fullwork:\r\n referer = f\"https://archiveofourown.org/works/{commentable.id}\"\r\n else:\r\n referer = f\"https://archiveofourown.org/chapters/{commentable.id}\"\r\n \r\n pseud_id = get_pseud_id(commentable, session, pseud)\r\n if pseud_id is None:\r\n raise PseudError(\"Couldn't find your pseud's id\")\r\n \r\n data.update({\r\n \"authenticity_token\": at,\r\n \"comment[pseud_id]\": pseud_id,\r\n \"comment[comment_content]\": comment_text,\r\n })\r\n \r\n else:\r\n if email == \"\" or name == \"\":\r\n raise ValueError(\"You need to specify both an email and a name!\")\r\n \r\n data.update({\r\n \"authenticity_token\": at,\r\n \"comment[email]\": email,\r\n \"comment[name]\": name,\r\n \"comment[comment_content]\": comment_text,\r\n })\r\n\r\n response = session.post(f\"https://archiveofourown.org/comments.js\", headers=headers, data=data)\r\n if response.status_code == 429:\r\n raise HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n if response.status_code == 404:\r\n if len(response.content) > 0:\r\n return response\r\n else:\r\n raise InvalidIdError(f\"Invalid {'work ID' if fullwork else 'chapter ID'}\")\r\n \r\n if response.status_code == 422:\r\n json = response.json()\r\n if \"errors\" in json:\r\n if \"auth_error\" in json[\"errors\"]:\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n raise UnexpectedResponseError(f\"Unexpected json received:\\n{str(json)}\")\r\n elif response.status_code == 200:\r\n raise DuplicateCommentError(\"You have already left this comment here\")\r\n\r\n raise UnexpectedResponseError(f\"Unexpected HTTP status code received ({response.status_code})\")\r\n\r\ndef delete_comment(comment, session):\r\n \"\"\"Deletes the specified comment\r\n\r\n Args:\r\n comment (AO3.Comment): Comment object\r\n session (AO3.Session): Session object\r\n\r\n Raises:\r\n PermissionError: You don't have permission to delete the comment\r\n utils.AuthError: Invalid auth token\r\n utils.UnexpectedResponseError: Unknown error\r\n \"\"\"\r\n \r\n if session is None or not session.is_authed:\r\n raise PermissionError(\"You don't have permission to do this\")\r\n \r\n if comment.authenticity_token is not None:\r\n at = comment.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n \r\n data = {\r\n \"authenticity_token\": at,\r\n \"_method\": \"delete\"\r\n }\r\n \r\n req = session.post(f\"https://archiveofourown.org/comments/{comment.id}\", data=data)\r\n if req.status_code == 429:\r\n raise HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n else:\r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n if \"auth error\" in soup.title.getText().lower():\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n else:\r\n error = soup.find(\"div\", {\"id\": \"main\"}).getText()\r\n if \"you don't have permission\" in error.lower():\r\n raise PermissionError(\"You don't have permission to do this\")\r\n \r\ndef kudos(work, session):\r\n \"\"\"Leave a 'kudos' in a specific work\r\n\r\n Args:\r\n work (Work): Work object\r\n\r\n Raises:\r\n utils.UnexpectedResponseError: Unexpected response received\r\n utils.InvalidIdError: Invalid ID (work doesn't exist)\r\n utils.AuthError: Invalid authenticity token\r\n\r\n Returns:\r\n bool: True if successful, False if you already left kudos there\r\n \"\"\"\r\n \r\n if work.authenticity_token is not None:\r\n at = work.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n data = {\r\n \"authenticity_token\": at,\r\n \"kudo[commentable_id]\": work.id,\r\n \"kudo[commentable_type]\": \"Work\"\r\n }\r\n headers = {\r\n \"x-csrf-token\": work.authenticity_token,\r\n \"x-requested-with\": \"XMLHttpRequest\",\r\n \"referer\": f\"https://archiveofourown.org/work/{work.id}\"\r\n }\r\n response = session.post(\"https://archiveofourown.org/kudos.js\", headers=headers, data=data)\r\n if response.status_code == 429:\r\n raise HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n \r\n if response.status_code == 201:\r\n return True # Success\r\n elif response.status_code == 422:\r\n json = response.json()\r\n if \"errors\" in json:\r\n if \"auth_error\" in json[\"errors\"]:\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n elif \"user_id\" in json[\"errors\"] or \"ip_address\" in json[\"errors\"]:\r\n return False # User has already left kudos\r\n elif \"no_commentable\" in json[\"errors\"]:\r\n raise InvalidIdError(\"Invalid ID\")\r\n raise UnexpectedResponseError(f\"Unexpected json received:\\n\"+str(json))\r\n else:\r\n raise UnexpectedResponseError(f\"Unexpected HTTP status code received ({response.status_code})\")\r\n \r\ndef subscribe(subscribable, worktype, session, unsubscribe=False, subid=None):\r\n \"\"\"Subscribes to a work. Be careful, you can subscribe to a work multiple times\r\n\r\n Args:\r\n subscribable (Work/Series/User): AO3 object\r\n worktype (str): Type of the work (Series/Work/User)\r\n session (AO3.Session): Session object\r\n unsubscribe (bool, optional): Unsubscribe instead of subscribing. Defaults to False.\r\n subid (str/int, optional): Subscription ID, used when unsubscribing. Defaults to None.\r\n\r\n Raises:\r\n AuthError: Invalid auth token\r\n AuthError: Invalid session\r\n InvalidIdError: Invalid ID / worktype\r\n InvalidIdError: Invalid subid\r\n \"\"\"\r\n \r\n if session is None: session = subscribable.session\r\n if session is None or not session.is_authed:\r\n raise AuthError(\"Invalid session\")\r\n \r\n if subscribable.authenticity_token is not None:\r\n at = subscribable.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n \r\n data = {\r\n \"authenticity_token\": at,\r\n \"subscription[subscribable_id]\": subscribable.id,\r\n \"subscription[subscribable_type]\": worktype.capitalize()\r\n }\r\n\r\n url = f\"https://archiveofourown.org/users/{session.username}/subscriptions\"\r\n if unsubscribe:\r\n if subid is None:\r\n raise InvalidIdError(\"When unsubscribing, subid cannot be None\")\r\n url += f\"/{subid}\"\r\n data[\"_method\"] = \"delete\"\r\n req = session.session.post(url, data=data, allow_redirects=False)\r\n if unsubscribe:\r\n return req\r\n if req.status_code == 302:\r\n if req.headers[\"Location\"] == AO3_AUTH_ERROR_URL:\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n else:\r\n raise InvalidIdError(f\"Invalid ID / worktype\")\r\n\r\ndef bookmark(bookmarkable, session=None, notes=\"\", tags=None, collections=None, private=False, recommend=False, pseud=None):\r\n \"\"\"Adds a bookmark to a work/series. Be careful, you can bookmark a work multiple times\r\n\r\n Args:\r\n bookmarkable (Work/Series): AO3 object\r\n session (AO3.Session): Session object\r\n notes (str, optional): Bookmark notes. Defaults to \"\".\r\n tags (list, optional): What tags to add. Defaults to None.\r\n collections (list, optional): What collections to add this bookmark to. Defaults to None.\r\n private (bool, optional): Whether this bookmark should be private. Defaults to False.\r\n recommend (bool, optional): Whether to recommend this bookmark. Defaults to False.\r\n pseud (str, optional): What pseud to add the bookmark under. Defaults to default pseud.\r\n \"\"\"\r\n \r\n if session is None: session = bookmarkable.session\r\n if session == None or not session.is_authed:\r\n raise AuthError(\"Invalid session\")\r\n \r\n if bookmarkable.authenticity_token is not None:\r\n at = bookmarkable.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n \r\n if tags is None: tags = []\r\n if collections is None: collections = [] \r\n \r\n pseud_id = get_pseud_id(bookmarkable, session, pseud)\r\n if pseud_id is None:\r\n raise PseudError(\"Couldn't find your pseud's id\") \r\n \r\n data = {\r\n \"authenticity_token\": at,\r\n \"bookmark[pseud_id]\": pseud_id,\r\n \"bookmark[tag_string]\": \",\".join(tags), \r\n \"bookmark[collection_names]\": \",\".join(collections),\r\n \"bookmark[private]\": int(private),\r\n \"bookmark[rec]\" : int(recommend),\r\n \"commit\": \"Create\"\r\n } \r\n \r\n if notes != \"\": data[\"bookmark[bookmarker_notes]\"] = notes\r\n \r\n url = url_join(bookmarkable.url, \"bookmarks\")\r\n req = session.session.post(url, data=data, allow_redirects=False)\r\n handle_bookmark_errors(req)\r\n \r\ndef delete_bookmark(bookmarkid, session, auth_token=None):\r\n \"\"\"Remove a bookmark from the work/series\r\n\r\n Args:\r\n bookmarkid (Work/Series): AO3 object\r\n session (AO3.Session): Session object\r\n auth_token (str, optional): Authenticity token. Defaults to None.\r\n \"\"\"\r\n if session == None or not session.is_authed:\r\n raise AuthError(\"Invalid session\")\r\n \r\n data = {\r\n \"authenticity_token\": session.authenticity_token if auth_token is None else auth_token,\r\n \"_method\": \"delete\"\r\n }\r\n \r\n url = f\"https://archiveofourown.org/bookmarks/{bookmarkid}\"\r\n req = session.session.post(url, data=data, allow_redirects=False)\r\n handle_bookmark_errors(req)\r\n \r\ndef handle_bookmark_errors(request):\r\n if request.status_code == 302:\r\n if request.headers[\"Location\"] == AO3_AUTH_ERROR_URL:\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n else:\r\n if request.status_code == 200:\r\n soup = BeautifulSoup(request.content, \"lxml\")\r\n error_div = soup.find(\"div\", {\"id\": \"error\", \"class\": \"error\"})\r\n if error_div is None:\r\n raise UnexpectedResponseError(\"An unknown error occurred\")\r\n \r\n errors = [item.getText() for item in error_div.findAll(\"li\")]\r\n if len(errors) == 0:\r\n raise BookmarkError(\"An unknown error occurred\")\r\n raise BookmarkError(\"Error(s) creating bookmark:\" + \" \".join(errors))\r\n\r\n raise UnexpectedResponseError(f\"Unexpected HTTP status code received ({request.status_code})\")\r\n\r\ndef get_pseud_id(ao3object, session=None, specified_pseud=None):\r\n if session is None:\r\n session = ao3object.session\r\n if session is None or not session.is_authed:\r\n raise AuthError(\"Invalid session\")\r\n \r\n soup = session.request(ao3object.url) \r\n pseud = soup.find(\"input\", {\"name\": re.compile(\".+\\\\[pseud_id\\\\]\")})\r\n if pseud is None:\r\n pseud = soup.find(\"select\", {\"name\": re.compile(\".+\\\\[pseud_id\\\\]\")})\r\n if pseud is None:\r\n return None\r\n pseud_id = None\r\n if specified_pseud:\r\n for option in pseud.findAll(\"option\"):\r\n if option.string == specified_pseud:\r\n pseud_id = option.attrs[\"value\"]\r\n break\r\n else:\r\n for option in pseud.findAll(\"option\"):\r\n if \"selected\" in option.attrs and option.attrs[\"selected\"] == \"selected\":\r\n pseud_id = option.attrs[\"value\"]\r\n break\r\n else:\r\n pseud_id = pseud.attrs[\"value\"]\r\n return pseud_id\r\n\r\ndef collect(collectable, session, collections):\r\n \"\"\"Invites a work to a collection. Be careful, you can collect a work multiple times\r\n\r\n Args:\r\n work (Work): Work object\r\n session (AO3.Session): Session object\r\n collections (list, optional): What collections to add this work to. Defaults to None.\r\n \"\"\"\r\n \r\n if session is None: session = collectable.session\r\n if session == None or not session.is_authed:\r\n raise AuthError(\"Invalid session\")\r\n \r\n if collectable.authenticity_token is not None:\r\n at = collectable.authenticity_token\r\n else:\r\n at = session.authenticity_token\r\n \r\n if collections is None: collections = [] \r\n \r\n data = {\r\n \"authenticity_token\": at,\r\n \"collection_names\": \",\".join(collections),\r\n \"commit\": \"Add\"\r\n }\r\n \r\n url = url_join(collectable.url, \"collection_items\")\r\n req = session.session.post(url, data=data, allow_redirects=True)\r\n \r\n if req.status_code == 302:\r\n if req.headers[\"Location\"] == AO3_AUTH_ERROR_URL:\r\n raise AuthError(\"Invalid authentication token. Try calling session.refresh_auth_token()\")\r\n elif req.status_code == 200:\r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n notice_div = soup.find(\"div\", {\"class\": \"notice\"})\r\n \r\n error_div = soup.find(\"div\", {\"class\": \"error\"})\r\n \r\n if error_div is None and notice_div is None:\r\n raise UnexpectedResponseError(\"An unknown error occurred\")\r\n\r\n if error_div is not None:\r\n errors = [item.getText() for item in error_div.findAll(\"ul\")]\r\n \r\n if len(errors) == 0:\r\n raise CollectError(\"An unknown error occurred\")\r\n \r\n raise CollectError(\"We couldn't add your submission to the following collection(s): \" + \" \".join(errors)) \r\n else:\r\n raise UnexpectedResponseError(f\"Unexpected HTTP status code received ({req.status_code})\")\r\n" }, { "path": "AO3/works.py", "content": "import warnings\r\nfrom datetime import datetime\r\nfrom functools import cached_property\r\n\r\nfrom bs4 import BeautifulSoup\r\n\r\nfrom . import threadable, utils\r\nfrom .chapters import Chapter\r\nfrom .comments import Comment\r\nfrom .requester import requester\r\nfrom .users import User\r\n\r\n\r\nclass Work:\r\n \"\"\"\r\n AO3 work object\r\n \"\"\"\r\n\r\n def __init__(self, workid, session=None, load=True, load_chapters=True):\r\n \"\"\"Creates a new AO3 work object\r\n\r\n Args:\r\n workid (int): AO3 work ID\r\n session (AO3.Session, optional): Used to access restricted works\r\n load (bool, optional): If true, the work is loaded on initialization. Defaults to True.\r\n load_chapters (bool, optional): If false, chapter text won't be parsed, and Work.load_chapters() will have to be called. Defaults to True.\r\n\r\n Raises:\r\n utils.InvalidIdError: Raised if the work wasn't found\r\n \"\"\"\r\n\r\n self._session = session\r\n self.chapters = []\r\n self.id = workid\r\n self._soup = None\r\n if load:\r\n self.reload(load_chapters)\r\n \r\n def __repr__(self):\r\n try:\r\n return f\"\"\r\n except:\r\n return f\"\"\r\n \r\n def __eq__(self, other):\r\n return isinstance(other, __class__) and other.id == self.id\r\n \r\n def __getstate__(self):\r\n d = {}\r\n for attr in self.__dict__:\r\n if isinstance(self.__dict__[attr], BeautifulSoup):\r\n d[attr] = (self.__dict__[attr].encode(), True)\r\n else:\r\n d[attr] = (self.__dict__[attr], False)\r\n return d\r\n \r\n def __setstate__(self, d):\r\n for attr in d:\r\n value, issoup = d[attr]\r\n if issoup:\r\n self.__dict__[attr] = BeautifulSoup(value, \"lxml\")\r\n else:\r\n self.__dict__[attr] = value\r\n \r\n @threadable.threadable\r\n def reload(self, load_chapters=True):\r\n \"\"\"\r\n Loads information about this work.\r\n This function is threadable.\r\n \r\n Args:\r\n load_chapters (bool, optional): If false, chapter text won't be parsed, and Work.load_chapters() will have to be called. Defaults to True.\r\n \"\"\"\r\n \r\n for attr in self.__class__.__dict__:\r\n if isinstance(getattr(self.__class__, attr), cached_property):\r\n if attr in self.__dict__:\r\n delattr(self, attr)\r\n \r\n self._soup = self.request(f\"https://archiveofourown.org/works/{self.id}?view_adult=true&view_full_work=true\")\r\n if \"Error 404\" in self._soup.find(\"h2\", {\"class\", \"heading\"}).text:\r\n raise utils.InvalidIdError(\"Cannot find work\")\r\n if load_chapters:\r\n self.load_chapters()\r\n \r\n def set_session(self, session):\r\n \"\"\"Sets the session used to make requests for this work\r\n\r\n Args:\r\n session (AO3.Session/AO3.GuestSession): session object\r\n \"\"\"\r\n \r\n self._session = session \r\n\r\n def load_chapters(self):\r\n \"\"\"Loads chapter objects for each one of this work's chapters\r\n \"\"\"\r\n \r\n self.chapters = []\r\n chapters_div = self._soup.find(attrs={\"id\": \"chapters\"})\r\n if chapters_div is None:\r\n return\r\n \r\n if self.nchapters > 1:\r\n for n in range(1, self.nchapters+1):\r\n chapter = chapters_div.find(\"div\", {\"id\": f\"chapter-{n}\"})\r\n if chapter is None:\r\n continue\r\n chapter.extract()\r\n preface_group = chapter.find(\"div\", {\"class\": (\"chapter\", \"preface\", \"group\")})\r\n if preface_group is None:\r\n continue\r\n title = preface_group.find(\"h3\", {\"class\": \"title\"})\r\n if title is None:\r\n continue\r\n id_ = int(title.a[\"href\"].split(\"/\")[-1])\r\n c = Chapter(id_, self, self._session, False)\r\n c._soup = chapter\r\n self.chapters.append(c)\r\n else:\r\n c = Chapter(None, self, self._session, False)\r\n c._soup = chapters_div\r\n self.chapters.append(c)\r\n \r\n def get_images(self):\r\n \"\"\"Gets all images from this work\r\n\r\n Raises:\r\n utils.UnloadedError: Raises this error if the work isn't loaded\r\n\r\n Returns:\r\n dict: key = chapter_n; value = chapter.get_images()\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n chapters = {}\r\n for chapter in self.chapters:\r\n images = chapter.get_images()\r\n if len(images) != 0:\r\n chapters[chapter.number] = images\r\n return chapters\r\n \r\n def download(self, filetype=\"PDF\"):\r\n \"\"\"Downloads this work\r\n\r\n Args:\r\n filetype (str, optional): Desired filetype. Defaults to \"PDF\".\r\n Known filetypes are: AZW3, EPUB, HTML, MOBI, PDF. \r\n\r\n Raises:\r\n utils.DownloadError: Raised if there was an error with the download\r\n utils.UnexpectedResponseError: Raised if the filetype is not available for download\r\n\r\n Returns:\r\n bytes: File content\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n download_btn = self._soup.find(\"li\", {\"class\": \"download\"})\r\n for download_type in download_btn.findAll(\"li\"):\r\n if download_type.a.getText() == filetype.upper():\r\n url = f\"https://archiveofourown.org/{download_type.a.attrs['href']}\"\r\n req = self.get(url)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n if not req.ok:\r\n raise utils.DownloadError(\"An error occurred while downloading the work\")\r\n return req.content\r\n raise utils.UnexpectedResponseError(f\"Filetype '{filetype}' is not available for download\")\r\n \r\n @threadable.threadable\r\n def download_to_file(self, filename, filetype=\"PDF\"):\r\n \"\"\"Downloads this work and saves it in the specified file.\r\n This function is threadable.\r\n\r\n Args:\r\n filename (str): Name of the resulting file\r\n filetype (str, optional): Desired filetype. Defaults to \"PDF\".\r\n Known filetypes are: AZW3, EPUB, HTML, MOBI, PDF.\r\n\r\n Raises:\r\n utils.DownloadError: Raised if there was an error with the download\r\n utils.UnexpectedResponseError: Raised if the filetype is not available for download\r\n \"\"\"\r\n with open(filename, \"wb\") as file:\r\n file.write(self.download(filetype))\r\n \r\n @property\r\n def metadata(self):\r\n metadata = {}\r\n normal_fields = (\r\n \"bookmarks\", \r\n \"categories\",\r\n \"nchapters\",\r\n \"characters\",\r\n \"complete\",\r\n \"comments\",\r\n \"expected_chapters\",\r\n \"fandoms\",\r\n \"hits\",\r\n \"kudos\",\r\n \"language\",\r\n \"rating\",\r\n \"relationships\",\r\n \"restricted\",\r\n \"status\",\r\n \"summary\",\r\n \"tags\",\r\n \"title\",\r\n \"warnings\",\r\n \"id\",\r\n \"words\",\r\n \"collections\"\r\n )\r\n string_fields = (\r\n \"date_edited\",\r\n \"date_published\",\r\n \"date_updated\",\r\n )\r\n \r\n for field in string_fields:\r\n try:\r\n metadata[field] = str(getattr(self, field))\r\n except AttributeError:\r\n pass\r\n \r\n for field in normal_fields:\r\n try:\r\n metadata[field] = getattr(self, field)\r\n except AttributeError:\r\n pass\r\n \r\n try:\r\n metadata[\"authors\"] = list(map(lambda author: author.username, self.authors))\r\n except AttributeError:\r\n pass\r\n try:\r\n metadata[\"series\"] = list(map(lambda series: series.name, self.series))\r\n except AttributeError:\r\n pass\r\n try:\r\n metadata[\"chapter_titles\"] = list(map(lambda chapter: chapter.title, self.chapters))\r\n except AttributeError:\r\n pass\r\n\r\n return metadata\r\n \r\n def get_comments(self, maximum=None):\r\n \"\"\"Returns a list of all threads of comments in the work. This operation can take a very long time.\r\n Because of that, it is recomended that you set a maximum number of comments. \r\n Duration: ~ (0.13 * n_comments) seconds or 2.9 seconds per comment page\r\n\r\n Args:\r\n maximum (int, optional): Maximum number of comments to be returned. None -> No maximum\r\n\r\n Raises:\r\n ValueError: Invalid chapter number\r\n IndexError: Invalid chapter number\r\n utils.UnloadedError: Work isn't loaded\r\n\r\n Returns:\r\n list: List of comments\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n url = f\"https://archiveofourown.org/works/{self.id}?page=%d&show_comments=true&view_adult=true&view_full_work=true\"\r\n soup = self.request(url%1)\r\n \r\n pages = 0\r\n div = soup.find(\"div\", {\"id\": \"comments_placeholder\"})\r\n ol = div.find(\"ol\", {\"class\": \"pagination actions\"})\r\n if ol is None:\r\n pages = 1\r\n else:\r\n for li in ol.findAll(\"li\"):\r\n if li.getText().isdigit():\r\n pages = int(li.getText()) \r\n \r\n comments = []\r\n for page in range(pages):\r\n if page != 0:\r\n soup = self.request(url%(page+1))\r\n ol = soup.find(\"ol\", {\"class\": \"thread\"})\r\n for li in ol.findAll(\"li\", {\"role\": \"article\"}, recursive=False):\r\n if maximum is not None and len(comments) >= maximum:\r\n return comments\r\n id_ = int(li.attrs[\"id\"][8:])\r\n \r\n header = li.find(\"h4\", {\"class\": (\"heading\", \"byline\")})\r\n if header is None or header.a is None:\r\n author = None\r\n else:\r\n author = User(str(header.a.text), self._session, False)\r\n \r\n if li.blockquote is not None:\r\n text = li.blockquote.getText()\r\n else:\r\n text = \"\" \r\n \r\n comment = Comment(id_, self, session=self._session, load=False) \r\n setattr(comment, \"authenticity_token\", self.authenticity_token)\r\n setattr(comment, \"author\", author)\r\n setattr(comment, \"text\", text)\r\n comment._thread = None\r\n comments.append(comment)\r\n return comments\r\n \r\n @threadable.threadable\r\n def subscribe(self):\r\n \"\"\"Subscribes to this work.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only subscribe to a work using an authenticated session\")\r\n \r\n utils.subscribe(self, \"Work\", self._session)\r\n \r\n @threadable.threadable\r\n def unsubscribe(self):\r\n \"\"\"Unubscribes from this user.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if not self.is_subscribed:\r\n raise Exception(\"You are not subscribed to this work\")\r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only unsubscribe from a work using an authenticated session\")\r\n \r\n utils.subscribe(self, \"Work\", self._session, True, self._sub_id)\r\n \r\n @cached_property\r\n def text(self):\r\n \"\"\"This work's text\"\"\"\r\n \r\n text = \"\"\r\n for chapter in self.chapters:\r\n text += chapter.text\r\n text += \"\\n\"\r\n return text\r\n \r\n @cached_property\r\n def authenticity_token(self):\r\n \"\"\"Token used to take actions that involve this work\"\"\"\r\n \r\n if not self.loaded:\r\n return None\r\n \r\n token = self._soup.find(\"meta\", {\"name\": \"csrf-token\"})\r\n return token[\"content\"]\r\n \r\n @cached_property\r\n def is_subscribed(self):\r\n \"\"\"True if you're subscribed to this work\"\"\"\r\n \r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only get a user ID using an authenticated session\")\r\n \r\n ul = self._soup.find(\"ul\", {\"class\": \"work navigation actions\"})\r\n input_ = ul.find(\"li\", {\"class\": \"subscribe\"}).find(\"input\", {\"name\": \"commit\", \"value\": \"Unsubscribe\"})\r\n return input_ is not None\r\n \r\n @cached_property\r\n def _sub_id(self):\r\n \"\"\"Returns the subscription ID. Used for unsubscribing\"\"\"\r\n \r\n if self._session is None or not self._session.is_authed:\r\n raise utils.AuthError(\"You can only get a user ID using an authenticated session\")\r\n \r\n ul = self._soup.find(\"ul\", {\"class\": \"work navigation actions\"})\r\n id_ = ul.find(\"li\", {\"class\": \"subscribe\"}).form.attrs[\"action\"].split(\"/\")[-1]\r\n return int(id_)\r\n \r\n @threadable.threadable\r\n def leave_kudos(self):\r\n \"\"\"Leave a \"kudos\" in this work.\r\n This function is threadable.\r\n\r\n Raises:\r\n utils.UnexpectedResponseError: Unexpected response received\r\n utils.InvalidIdError: Invalid ID (work doesn't exist)\r\n utils.AuthError: Invalid session or authenticity token\r\n\r\n Returns:\r\n bool: True if successful, False if you already left kudos there\r\n \"\"\"\r\n \r\n if self._session is None:\r\n raise utils.AuthError(\"Invalid session\")\r\n return utils.kudos(self, self._session)\r\n \r\n @threadable.threadable\r\n def comment(self, comment_text, email=\"\", name=\"\", pseud=None):\r\n \"\"\"Leaves a comment on this work.\r\n This function is threadable.\r\n\r\n Args:\r\n comment_text (str): Comment text\r\n email (str, optional): Email to add comment. Needed if not logged in.\r\n name (str, optional): Name to add comment under. Needed if not logged in.\r\n pseud (str, optional): Pseud to add the comment under. Defaults to default pseud.\r\n\r\n Raises:\r\n utils.UnloadedError: Couldn't load chapters\r\n utils.AuthError: Invalid session\r\n\r\n Returns:\r\n requests.models.Response: Response object\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n if self._session is None:\r\n raise utils.AuthError(\"Invalid session\")\r\n \r\n return utils.comment(self, comment_text, self._session, True, email=email, name=name, pseud=pseud)\r\n \r\n @threadable.threadable\r\n def bookmark(self, notes=\"\", tags=None, collections=None, private=False, recommend=False, pseud=None):\r\n \"\"\"Bookmarks this work\r\n This function is threadable\r\n\r\n Args:\r\n notes (str, optional): Bookmark notes. Defaults to \"\".\r\n tags (list, optional): What tags to add. Defaults to None.\r\n collections (list, optional): What collections to add this bookmark to. Defaults to None.\r\n private (bool, optional): Whether this bookmark should be private. Defaults to False.\r\n recommend (bool, optional): Whether to recommend this bookmark. Defaults to False.\r\n pseud (str, optional): What pseud to add the bookmark under. Defaults to default pseud.\r\n\r\n Raises:\r\n utils.UnloadedError: Work isn't loaded\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n if self._session is None:\r\n raise utils.AuthError(\"Invalid session\")\r\n \r\n utils.bookmark(self, self._session, notes, tags, collections, private, recommend, pseud)\r\n \r\n @threadable.threadable\r\n def delete_bookmark(self):\r\n \"\"\"Removes a bookmark from this work\r\n This function is threadable\r\n\r\n Raises:\r\n utils.UnloadedError: Work isn't loaded\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n if self._session is None:\r\n raise utils.AuthError(\"Invalid session\")\r\n \r\n if self._bookmarkid is None:\r\n raise utils.BookmarkError(\"You don't have a bookmark here\")\r\n \r\n utils.delete_bookmark(self._bookmarkid, self._session, self.authenticity_token)\r\n \r\n @threadable.threadable\r\n def collect(self, collections):\r\n \"\"\"Invites/collects this work to a collection or collections\r\n This function is threadable\r\n\r\n Args:\r\n collections (list): What collections to add this work to. Defaults to None.\r\n\r\n Raises:\r\n utils.UnloadedError: Work isn't loaded\r\n utils.AuthError: Invalid session\r\n \"\"\"\r\n \r\n if not self.loaded:\r\n raise utils.UnloadedError(\"Work isn't loaded. Have you tried calling Work.reload()?\")\r\n \r\n if self._session is None:\r\n raise utils.AuthError(\"Invalid session\")\r\n \r\n utils.collect(self, self._session, collections)\r\n \r\n @cached_property\r\n def _bookmarkid(self):\r\n form_div = self._soup.find(\"div\", {\"id\": \"bookmark-form\"})\r\n if form_div is None: \r\n return None\r\n if form_div.form is None:\r\n return None\r\n if \"action\" in form_div.form.attrs and form_div.form[\"action\"].startswith(\"/bookmarks\"):\r\n text = form_div.form[\"action\"].split(\"/\")[-1]\r\n if text.isdigit():\r\n return int(text)\r\n return None\r\n return None\r\n \r\n @property\r\n def loaded(self):\r\n \"\"\"Returns True if this work has been loaded\"\"\"\r\n return self._soup is not None\r\n \r\n @property\r\n def oneshot(self):\r\n \"\"\"Returns True if this work has only one chapter\"\"\"\r\n return self.nchapters == 1\r\n \r\n @cached_property\r\n def series(self):\r\n \"\"\"Returns the series this work belongs to\"\"\"\r\n \r\n from .series import Series\r\n dd = self._soup.find(\"dd\", {\"class\": \"series\"})\r\n if dd is None:\r\n return []\r\n \r\n s = []\r\n for span in dd.find_all(\"span\", {\"class\": \"position\"}):\r\n seriesid = int(span.a.attrs[\"href\"].split(\"/\")[-1])\r\n seriesname = span.a.getText()\r\n series = Series(seriesid, self._session, False)\r\n setattr(series, \"name\", seriesname)\r\n s.append(series)\r\n return s\r\n\r\n @cached_property\r\n def authors(self):\r\n \"\"\"Returns the list of the work's author\r\n\r\n Returns:\r\n list: list of authors\r\n \"\"\"\r\n\r\n from .users import User\r\n authors = self._soup.find_all(\"h3\", {\"class\": \"byline heading\"})\r\n if len(authors) == 0:\r\n return []\r\n formatted_authors = authors[0].text.replace(\"\\n\", \"\").split(\", \")\r\n author_list = []\r\n if authors is not None:\r\n for author in formatted_authors:\r\n user = User(author, load=False)\r\n author_list.append(user)\r\n\r\n return author_list\r\n\r\n @cached_property\r\n def nchapters(self):\r\n \"\"\"Returns the number of chapters of this work\r\n\r\n Returns:\r\n int: number of chapters\r\n \"\"\"\r\n \r\n chapters = self._soup.find(\"dd\", {\"class\": \"chapters\"})\r\n if chapters is not None:\r\n return int(self.str_format(chapters.string.split(\"/\")[0]))\r\n return 0\r\n \r\n @cached_property\r\n def expected_chapters(self):\r\n \"\"\"Returns the number of expected chapters for this work, or None if \r\n the author hasn't provided an expected number\r\n\r\n Returns:\r\n int: number of chapters\r\n \"\"\"\r\n chapters = self._soup.find(\"dd\", {\"class\": \"chapters\"})\r\n if chapters is not None:\r\n n = self.str_format(chapters.string.split(\"/\")[-1])\r\n if n.isdigit():\r\n return int(n)\r\n return None\r\n \r\n @property\r\n def status(self):\r\n \"\"\"Returns the status of this work\r\n\r\n Returns:\r\n str: work status\r\n \"\"\"\r\n\r\n return \"Completed\" if self.nchapters == self.expected_chapters else \"Work in Progress\"\r\n\r\n @cached_property\r\n def hits(self):\r\n \"\"\"Returns the number of hits this work has\r\n\r\n Returns:\r\n int: number of hits\r\n \"\"\"\r\n\r\n hits = self._soup.find(\"dd\", {\"class\": \"hits\"})\r\n if hits is not None:\r\n return int(self.str_format(hits.string))\r\n return 0\r\n\r\n @cached_property\r\n def kudos(self):\r\n \"\"\"Returns the number of kudos this work has\r\n\r\n Returns:\r\n int: number of kudos\r\n \"\"\"\r\n\r\n kudos = self._soup.find(\"dd\", {\"class\": \"kudos\"})\r\n if kudos is not None:\r\n return int(self.str_format(kudos.string))\r\n return 0\r\n\r\n @cached_property\r\n def comments(self):\r\n \"\"\"Returns the number of comments this work has\r\n\r\n Returns:\r\n int: number of comments\r\n \"\"\"\r\n\r\n comments = self._soup.find(\"dd\", {\"class\": \"comments\"})\r\n if comments is not None:\r\n return int(self.str_format(comments.string))\r\n return 0\r\n \r\n @cached_property\r\n def restricted(self):\r\n \"\"\"Whether this is a restricted work or not\r\n \r\n Returns:\r\n int: True if work is restricted\r\n \"\"\"\r\n return self._soup.find(\"img\", {\"title\": \"Restricted\"}) is not None\r\n\r\n @cached_property\r\n def words(self):\r\n \"\"\"Returns the this work's word count\r\n\r\n Returns:\r\n int: number of words\r\n \"\"\"\r\n\r\n words = self._soup.find(\"dd\", {\"class\": \"words\"})\r\n if words is not None:\r\n return int(self.str_format(words.string))\r\n return 0\r\n\r\n @cached_property\r\n def language(self):\r\n \"\"\"Returns this work's language\r\n\r\n Returns:\r\n str: Language\r\n \"\"\"\r\n\r\n language = self._soup.find(\"dd\", {\"class\": \"language\"})\r\n if language is not None:\r\n return language.string.strip()\r\n else:\r\n return \"Unknown\"\r\n\r\n @cached_property\r\n def bookmarks(self):\r\n \"\"\"Returns the number of bookmarks this work has\r\n\r\n Returns:\r\n int: number of bookmarks\r\n \"\"\"\r\n\r\n bookmarks = self._soup.find(\"dd\", {\"class\": \"bookmarks\"})\r\n if bookmarks is not None:\r\n return int(self.str_format(bookmarks.string))\r\n return 0\r\n\r\n @cached_property\r\n def title(self):\r\n \"\"\"Returns the title of this work\r\n\r\n Returns:\r\n str: work title\r\n \"\"\"\r\n\r\n title = self._soup.find(\"div\", {\"class\": \"preface group\"})\r\n if title is not None:\r\n return str(title.h2.text.strip())\r\n return \"\"\r\n \r\n @cached_property\r\n def date_published(self):\r\n \"\"\"Returns the date this work was published\r\n\r\n Returns:\r\n datetime.date: publish date\r\n \"\"\"\r\n\r\n dp = self._soup.find(\"dd\", {\"class\": \"published\"}).string\r\n return datetime(*list(map(int, dp.split(\"-\"))))\r\n\r\n @cached_property\r\n def date_edited(self):\r\n \"\"\"Returns the date this work was last edited\r\n\r\n Returns:\r\n datetime.datetime: edit date\r\n \"\"\"\r\n\r\n download = self._soup.find(\"li\", {\"class\": \"download\"})\r\n if download is not None and download.ul is not None:\r\n timestamp = int(download.ul.a[\"href\"].split(\"=\")[-1])\r\n return datetime.fromtimestamp(timestamp)\r\n return datetime(self.date_published)\r\n\r\n @cached_property\r\n def date_updated(self):\r\n \"\"\"Returns the date this work was last updated\r\n\r\n Returns:\r\n datetime.datetime: update date\r\n \"\"\"\r\n update = self._soup.find(\"dd\", {\"class\": \"status\"})\r\n if update is not None:\r\n split = update.string.split(\"-\")\r\n return datetime(*list(map(int, split)))\r\n return self.date_published\r\n \r\n @cached_property\r\n def tags(self):\r\n \"\"\"Returns all the work's tags\r\n\r\n Returns:\r\n list: List of tags\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"freeform tags\"})\r\n tags = []\r\n if html is not None:\r\n for tag in html.find_all(\"li\"):\r\n tags.append(tag.a.string)\r\n return tags\r\n\r\n @cached_property\r\n def characters(self):\r\n \"\"\"Returns all the work's characters\r\n\r\n Returns:\r\n list: List of characters\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"character tags\"})\r\n characters = []\r\n if html is not None:\r\n for character in html.find_all(\"li\"):\r\n characters.append(character.a.string)\r\n return characters\r\n\r\n @cached_property\r\n def relationships(self):\r\n \"\"\"Returns all the work's relationships\r\n\r\n Returns:\r\n list: List of relationships\r\n \"\"\"\r\n \r\n html = self._soup.find(\"dd\", {\"class\": \"relationship tags\"})\r\n relationships = []\r\n if html is not None:\r\n for relationship in html.find_all(\"li\"):\r\n relationships.append(relationship.a.string)\r\n return relationships\r\n\r\n @cached_property\r\n def fandoms(self):\r\n \"\"\"Returns all the work's fandoms\r\n\r\n Returns:\r\n list: List of fandoms\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"fandom tags\"})\r\n fandoms = []\r\n if html is not None:\r\n for fandom in html.find_all(\"li\"):\r\n fandoms.append(fandom.a.string)\r\n return fandoms\r\n\r\n @cached_property\r\n def categories(self):\r\n \"\"\"Returns all the work's categories\r\n\r\n Returns:\r\n list: List of categories\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"category tags\"})\r\n categories = []\r\n if html is not None:\r\n for category in html.find_all(\"li\"):\r\n categories.append(category.a.string)\r\n return categories\r\n\r\n @cached_property\r\n def warnings(self):\r\n \"\"\"Returns all the work's warnings\r\n\r\n Returns:\r\n list: List of warnings\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"warning tags\"})\r\n warnings = []\r\n if html is not None:\r\n for warning in html.find_all(\"li\"):\r\n warnings.append(warning.a.string)\r\n return warnings\r\n\r\n @cached_property\r\n def rating(self):\r\n \"\"\"Returns this work's rating\r\n\r\n Returns:\r\n str: Rating\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"rating tags\"})\r\n if html is not None:\r\n rating = html.a.string\r\n return rating\r\n return None\r\n\r\n @cached_property\r\n def summary(self):\r\n \"\"\"Returns this work's summary\r\n\r\n Returns:\r\n str: Summary\r\n \"\"\"\r\n\r\n div = self._soup.find(\"div\", {\"class\": \"preface group\"})\r\n if div is None:\r\n return \"\"\r\n html = div.find(\"blockquote\", {\"class\": \"userstuff\"})\r\n if html is None:\r\n return \"\"\r\n return str(BeautifulSoup.getText(html))\r\n \r\n @cached_property\r\n def start_notes(self):\r\n \"\"\"Text from this work's start notes\"\"\"\r\n notes = self._soup.find(\"div\", {\"class\": \"notes module\"})\r\n if notes is None:\r\n return \"\"\r\n text = \"\"\r\n for p in notes.findAll(\"p\"):\r\n text += p.getText().strip() + \"\\n\"\r\n return text\r\n\r\n @cached_property\r\n def end_notes(self):\r\n \"\"\"Text from this work's end notes\"\"\"\r\n notes = self._soup.find(\"div\", {\"id\": \"work_endnotes\"})\r\n if notes is None:\r\n return \"\"\r\n text = \"\"\r\n for p in notes.findAll(\"p\"):\r\n text += p.getText() + \"\\n\"\r\n return text\r\n \r\n @cached_property\r\n def url(self):\r\n \"\"\"Returns the URL to this work\r\n\r\n Returns:\r\n str: work URL\r\n \"\"\" \r\n\r\n return f\"https://archiveofourown.org/works/{self.id}\"\r\n\r\n @cached_property\r\n def complete(self):\r\n \"\"\"\r\n Return True if the work is complete\r\n\r\n Retuns:\r\n bool: True if a work is complete\r\n \"\"\"\r\n\r\n chapterStatus = self._soup.find(\"dd\", {\"class\": \"chapters\"}).string.split(\"/\")\r\n return chapterStatus[0] == chapterStatus[1]\r\n \r\n @cached_property\r\n def collections(self):\r\n \"\"\"Returns all the collections the work belongs to\r\n\r\n Returns:\r\n list: List of collections\r\n \"\"\"\r\n\r\n html = self._soup.find(\"dd\", {\"class\": \"collections\"})\r\n collections = []\r\n if html is not None:\r\n for collection in html.find_all(\"a\"):\r\n collections.append(collection.get_text())\r\n return collections\r\n \r\n def get(self, *args, **kwargs):\r\n \"\"\"Request a web page and return a Response object\"\"\" \r\n \r\n if self._session is None:\r\n req = requester.request(\"get\", *args, **kwargs)\r\n else:\r\n req = requester.request(\"get\", *args, **kwargs, session=self._session.session)\r\n if req.status_code == 429:\r\n raise utils.HTTPError(\"We are being rate-limited. Try again in a while or reduce the number of requests\")\r\n return req\r\n\r\n def request(self, url):\r\n \"\"\"Request a web page and return a BeautifulSoup object.\r\n\r\n Args:\r\n url (str): Url to request\r\n\r\n Returns:\r\n bs4.BeautifulSoup: BeautifulSoup object representing the requested page's html\r\n \"\"\"\r\n\r\n req = self.get(url)\r\n if len(req.content) > 650000:\r\n warnings.warn(\"This work is very big and might take a very long time to load\")\r\n soup = BeautifulSoup(req.content, \"lxml\")\r\n return soup\r\n\r\n @staticmethod\r\n def str_format(string):\r\n \"\"\"Formats a given string\r\n\r\n Args:\r\n string (str): String to format\r\n\r\n Returns:\r\n str: Formatted string\r\n \"\"\"\r\n\r\n return string.replace(\",\", \"\")\r\n" }, { "path": "LICENSE", "content": "MIT License\r\n\r\nCopyright (c) 2019 Francisco Patrcio Rodrigues\r\n\r\nPermission is hereby granted, free of charge, to any person obtaining a copy\r\nof this software and associated documentation files (the \"Software\"), to deal\r\nin the Software without restriction, including without limitation the rights\r\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\r\ncopies of the Software, and to permit persons to whom the Software is\r\nfurnished to do so, subject to the following conditions:\r\n\r\nThe above copyright notice and this permission notice shall be included in all\r\ncopies or substantial portions of the Software.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\r\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\r\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\r\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\r\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\r\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r\nSOFTWARE." }, { "path": "README.md", "content": "[![Documentation Status](https://readthedocs.org/projects/ao3-api/badge/?version=latest)](https://ao3-api.readthedocs.io/en/latest/?badge=latest)\r\n\r\n# AO3 API\r\n\r\nThis is an unofficial API that lets you access some of AO3's (archiveofourown.org) data through Python.\r\n\r\n## Installation\r\n\r\nUse the package manager [pip](https://pip.pypa.io/en/stable/) to install AO3 API.\r\n\r\n```bash\r\npip install ao3_api\r\n```\r\n\r\n# Github\r\n\r\nhttps://github.com/wendytg/ao3_api\r\n\r\n\r\n# Usage\r\n\r\nThis package is divided in 9 core modules: works, chapters, users, series, search, session, comments, extra, and utils.\r\n\r\n## Works\r\n\r\nOne of the most basic things you might want to do with this package is loading a work and checking its statistics and information. To do that, you'll need the `AO3.Work` class.\r\n\r\nWe start by finding the _workid_ of the work we want to load. We do that either by using `AO3.utils.workid_from_url(url)` or by just looking at the url ourselves. Let's take a look:\r\n\r\n```py3\r\nimport AO3\r\n\r\nurl = \"https://archiveofourown.org/works/14392692/chapters/33236241\"\r\nworkid = AO3.utils.workid_from_url(url)\r\nprint(f\"Work ID: {workid}\")\r\nwork = AO3.Work(workid)\r\nprint(f\"Chapters: {work.nchapters}\")\r\n```\r\n\r\nAfter running this snippet, we get the output:\r\n\r\n```\r\nWork ID: 14392692\r\nChapters: 46\r\n```\r\n\r\nIt's important to note that some works may not be accessible to guest users, and in this case you will get 0 chapters as an output, and the error `AO3.utils.AuthError: This work is only available to registered users of the Archive` if you try to load it. Nontheless, we can still do a lot more with this Work object: Lets try to get the first 20 words of the second chapter.\r\n\r\n```py3\r\nimport AO3\r\n\r\nwork = AO3.Work(14392692)\r\n\r\nprint(work.chapters[1].title) # Second chapter name\r\ntext = work.chapters[1].text # Second chapter text\r\nprint(' '.join(text.split(\" \")[:20]))\r\n```\r\n\r\n```\r\nWhat Branches Grow Meaning\r\nDecember 27, 2018\r\n \r\nChristmas sucked this year, and Shouto’s got the black eye to prove it.\r\nThings had started out well enough,\r\n```\r\n\r\nThe objects in work.chapters are of type `AO3.Chapter`. They have a lot of the same properties as a `Work` object would.\r\n\r\n\r\nAnother thing you can do with the work object is download the entire work as a pdf or e-book. At the moment you can download works as AZW3, EPUB, HTML, MOBI, and PDF files.\r\n\r\n```py3\r\nimport AO3\r\n\r\nwork = AO3.Work(14392692)\r\n\r\nwith open(f\"{work.title}.pdf\", \"wb\") as file:\r\n file.write(work.download(\"PDF\"))\r\n```\r\n\r\n\r\n__Advanced functionality__\r\n\r\nUsually, when you call the constructor for the `Work` class, all info about it is loaded in the `__init__()` function. However, this process takes quite some time (~1-1.5 seconds) and if you want to load a list of works from a series, for example, you might be waiting for upwards of 30 seconds. To avoid this problem, the `Work.reload()` function, called on initialization, is a \"threadable\" function, which means that if you call it with the argument `threaded=True`, it will return a `Thread` object and work in parallel, meaning you can load multiple works at the same time. Let's take a look at an implementation:\r\n\r\n```py3\r\nimport AO3\r\nimport time\r\n\r\nseries = AO3.Series(1295090)\r\n\r\nworks = []\r\nthreads = []\r\nstart = time.time()\r\nfor work in series.work_list:\r\n works.append(work)\r\n threads.append(work.reload(threaded=True))\r\nfor thread in threads:\r\n thread.join()\r\nprint(f\"Loaded {len(works)} works in {round(time.time()-start, 1)} seconds.\")\r\n```\r\n\r\n`Loaded 29 works in 2.2 seconds.`\r\n\r\nThe `load=False` inside the `Work` constructor makes sure we don't load the work as soon as we create an instance of the class. In the end, we iterate over every thread and wait for the last one to finish using `.join()`. Let's compare this method with the standard way of loading AO3 works:\r\n\r\n```py3\r\nimport AO3\r\nimport time\r\n\r\nseries = AO3.Series(1295090)\r\n\r\nworks = []\r\nstart = time.time()\r\nfor work in series.work_list:\r\n work.reload()\r\n works.append(work)\r\n\r\nprint(f\"Loaded {len(works)} works in {round(time.time()-start, 1)} seconds.\")\r\n```\r\n\r\n`Loaded 29 works in 21.6 seconds.`\r\n\r\nAs we can see, there is a significant performance increase. There are other functions in this package which have this functionality. To see if a function is \"threadable\", either use `hasattr(function, \"_threadable\")` or check its `__doc__` string.\r\n\r\nTo save even more time, if you're only interested in metadata, you can load a work with the `load_chapters` option set to False. Also, be aware that some functions (like `Series.work_list` or `Search.results`) might return semi-loaded `Work` objects. This means that no requests have been made to load this work (so you don't have access to chapter text, notes, etc...) but almost all of its metadata will already have been cached, and you might not need to call `Work.reload()` at all. \r\n\r\nThe last important information about the `Work` class is that most of its properties (like the number of bookmarks, kudos, the authors' names, etc...) are cached properties. That means that once you check them once, the value is stored and it won't ever change, even if those values change. To update these values, you will need to call `Work.reload()`. See the example below:\r\n\r\n```py3\r\nimport AO3\r\n\r\nsess = AO3.GuestSession()\r\nwork = AO3.Work(16721367, sess)\r\nprint(work.kudos)\r\nwork.leave_kudos()\r\nwork.reload()\r\nprint(work.kudos)\r\n```\r\n\r\n```\r\n392\r\n393\r\n```\r\n\r\n\r\n\r\n## Users\r\n\r\nAnother useful thing you might want to do is get information on who wrote which works / comments. For that, we use the `AO3.User` class.\r\n\r\n```py3\r\nimport AO3\r\n\r\nuser = AO3.User(\"bothersomepotato\")\r\nprint(user.url)\r\nprint(user.bio)\r\nprint(user.works) # Number of works published\r\n```\r\n\r\n```\r\nhttps://archiveofourown.org/users/bothersomepotato\r\nUniversity student, opening documents to write essays but writing this stuff instead. No regrets though. My Tumblr, come chat with -or yell at- me if you feel like it! :)\r\n2\r\n```\r\n\r\n\r\n## Search\r\n\r\nTo search for works, you can either use the `AO3.search()` function and parse the BeautifulSoup object returned yourself, or use the `AO3.Search` class to automatically do that for you.\r\n\r\n```py3\r\nimport AO3\r\nsearch = AO3.Search(any_field=\"Clarke Lexa\", word_count=AO3.utils.Constraint(5000, 15000))\r\nsearch.update()\r\nprint(search.total_results)\r\nfor result in search.results:\r\n print(result)\r\n```\r\n\r\n```\r\n3074\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n```\r\n\r\nYou can then use the workid to load one of the works you searched for. To get more then the first 20 works, change the page number using \r\n```py3\r\nsearch.page = 2\r\n```\r\n\r\n## Session\r\n\r\nA lot of actions you might want to take might require an AO3 account. If you already have one, you can access those actions using an AO3.Session object. You start by logging in using your username and password, and then you can use that object to access restricted content.\r\n\r\n```py3\r\nimport AO3\r\n\r\nsession = AO3.Session(\"username\", \"password\")\r\nprint(f\"Bookmarks: {session.bookmarks}\")\r\nsession.refresh_auth_token()\r\nprint(session.kudos(AO3.Work(18001499, load=False))\r\n```\r\n\r\n```\r\nBookmarks: 67\r\nTrue\r\n```\r\n\r\nWe successfully left kudos in a work and checked our bookmarks. The `session.refresh_auth_token()` is needed for some activities such as leaving kudos and comments. If it is expired or you forget to call this function, the error `AO3.utils.AuthError: Invalid authentication token. Try calling session.refresh_auth_token()` will be raised.\r\n\r\nYou can also comment / leave kudos in a work by calling `Work.leave_kudos()`/`Work.comment()` and provided you have instantiated that object with a session already (`AO3.Work(xxxxxx, session=sess)` or using `Work.set_session()`). This is probably the best way to do so because you will run into less authentication issues (as the work's authenticity token will be used instead).\r\n\r\nIf you would prefer to leave a comment or kudos anonymously, you can use an `AO3.GuestSession` in the same way you'd use a normal session, except you won't be able to check your bookmarks, subscriptions, etc. because you're not actually logged in.\r\n\r\n\r\n## Comments\r\n\r\nTo retrieve and process comment threads, you might want to look at the `Work.get_comments()` method. It returns all the comments in a specific chapter and their respective threads. You can then process them however you want. Let's take a look:\r\n\r\n```py3\r\nfrom time import time\r\n\r\nimport AO3\r\n\r\n\r\nwork = AO3.Work(24560008)\r\nwork.load_chapters()\r\nstart = time()\r\ncomments = work.get_comments(5)\r\nprint(f\"Loaded {len(comments)} comment threads in {round(time()-start, 1)} seconds\\n\")\r\nfor comment in comments:\r\n print(f\"Comment ID: {comment.id}\\nReplies: {len(comment.get_thread())}\")\r\n```\r\n\r\n```\r\nLoaded 5 comment threads in 1.8 seconds\r\n\r\nComment ID: 312237184\r\nReplies: 1\r\nComment ID: 312245032\r\nReplies: 1\r\nComment ID: 312257098\r\nReplies: 1\r\nComment ID: 312257860\r\nReplies: 1\r\nComment ID: 312285673\r\nReplies: 2\r\n```\r\n\r\nLoading comments takes a very long time so you should try and use it as little as possible. It also causes lots of requests to be sent to the AO3 servers, which might result in getting the error `utils.HTTPError: We are being rate-limited. Try again in a while or reduce the number of requests`. If that happens, you should try to space out your requests or reduce their number. There is also the option to enable request limiting using `AO3.utils.limit_requests()`, which make it so you can't make more than x requests in a certain time window.\r\nYou can also reply to comments using the `Comment.reply()` function, or delete one (if it's yours) using `Comment.delete()`.\r\n\r\n\r\n## Extra\r\n\r\nAO3.extra contains the the code to download some extra resources that are not core to the functionality of this package and don't change very often. One example would be the list of fandoms recognized by AO3.\r\nTo download a resource, simply use `AO3.extra.download(resource_name)`. To download every resource, you can use `AO3.extra.download_all()`. To see the list of available resources, use `AO3.extra.get_resources()`.\r\n\r\n\r\n# Contact info\r\n\r\nFor information or bug reports, please create an issue or start a discussion.\r\n\r\n\r\n# License\r\n[MIT](https://choosealicense.com/licenses/mit/)\r\n" }, { "path": "docs/index.md", "content": "# AO3 API\n\nThis is an unofficial python library that lets you access some of AO3's (archiveofourown.org) data using webscraping and some other tools.\n\n__Documentation__\n\nhttps://ao3-api.readthedocs.io\n\n__Source code repository and issue tracker__\n\nhttps://github.com/wendytg/ao3_api\n\n__License__\n\n[MIT](https://choosealicense.com/licenses/mit/)\n \n" }, { "path": "docs/install.md", "content": "# Installation\n\nYou can install this package using pip\n\n```pip install ao3_api```\n\nor by cloning the repository and building it from source.\n\n__Requirements__\n\n- BeautifulSoup4\n- Requests\n- LXML\n " }, { "path": "docs/use.md", "content": "# Usage\n\nThis package is divided in 9 core modules: works, chapters, users, series, search, session, comments, extra, and utils.\n\n## Works\n\nOne of the most basic things you might want to do with this package is loading a work and checking its statistics and informations. To do that, you'll need the `AO3.Work` class.\n\nWe start by finding the _workid_ of the work we want to load. We do that either by using `AO3.utils.workid_from_url(url)` or by just looking at the url ourselves. Let's take a look:\n\n```python\nimport AO3\n\nurl = \"https://archiveofourown.org/works/14392692/chapters/33236241\"\nworkid = AO3.utils.workid_from_url(url)\nprint(f\"Work ID: {workid}\")\nwork = AO3.Work(workid)\nprint(f\"Chapters: {work.nchapters}\")\n```\n\nAfter running this snippet, we get the output:\n\n```\nWork ID: 14392692\nChapters: 46\n```\n\nIt's important to note that some works may not be accessible to guest users, and in this case you will get 0 chapters as an output, and the error `AO3.utils.AuthError: This work is only available to registered users of the Archive` if you try to load it. Nontheless, we can still do a lot more with this Work object: Lets try to get the first 20 words of the second chapter.\n\n```python\nimport AO3\n\nwork = AO3.Work(14392692)\n\nprint(work.chapters[1].title) # Second chapter name\ntext = work.chapters[1].text # Second chapter text\nprint(' '.join(text.split(\" \")[:20]))\n```\n\n```\nWhat Branches Grow Meaning\nDecember 27, 2018\n\nChristmas sucked this year, and Shouto’s got the black eye to prove it.\nThings had started out well enough,\n```\n\nThe objects in work.chapters are of type `AO3.Chapter`. They have a lot of the same properties as a `Work` object would.\n\n\nAnother thing you can do with the work object is download the entire work as a pdf or e-book. At the moment you can download works as AZW3, EPUB, HTML, MOBI, and PDF files.\n\n```python\nimport AO3\n\nwork = AO3.Work(14392692)\n\nwith open(f\"{work.title}.pdf\", \"wb\") as file:\n file.write(work.download(\"PDF\"))\n```\n\n\n__Advanced functionality__\n\nUsually, when you call the constructor for the `Work` class, all info about it is loaded in the `__init__()` function. However, this process takes quite some time (~1-1.5 seconds) and if you want to load a list of works from a series, for example, you might be waiting for upwards of 30 seconds. To avoid this problem, the `Work.reload()` function, called on initialization, is a \"threadable\" function, which means that if you call it with the argument `threaded=True`, it will return a `Thread` object and work in parallel, meaning you can load multiple works at the same time. Let's take a look at an implementation:\n\n```python\nimport AO3\nimport time\n\nseries = AO3.Series(1295090)\n\nworks = []\nthreads = []\nstart = time.time()\nfor work in series.work_list:\n works.append(work)\n threads.append(work.reload(threaded=True))\nfor thread in threads:\n thread.join()\nprint(f\"Loaded {len(works)} works in {round(time.time()-start, 1)} seconds.\")\n```\n\n`Loaded 29 works in 2.2 seconds.`\n\nThe `load=False` inside the `Work` constructor makes sure we don't load the work as soon as we create an instance of the class. In the end, we iterate over every thread and wait for the last one to finish using `.join()`. Let's compare this method with the standard way of loading AO3 works:\n\n```python\nimport AO3\nimport time\n\nseries = AO3.Series(1295090)\n\nworks = []\nstart = time.time()\nfor work in series.work_list:\n work.reload()\n works.append(work)\n\nprint(f\"Loaded {len(works)} works in {round(time.time()-start, 1)} seconds.\")\n```\n\n`Loaded 29 works in 21.6 seconds.`\n\nAs we can see, there is a significant performance increase. There are other functions in this package which have this functionality. To see if a function is \"threadable\", either use `hasattr(function, \"_threadable\")` or check its `__doc__` string.\n\nTo save even more time, if you're only interested in metadata, you can load a work with the `load_chapters` option set to False. Also, be aware that some functions (like `Series.work_list` or `Search.results`) might return semi-loaded `Work` objects. This means that no requests have been made to load this work (so you don't have access to chapter text, notes, etc...) but almost all of its metadata will already have been cached, and you might not need to call `Work.reload()` at all. \n\nThe last important information about the `Work` class is that most of its properties (like the number of bookmarks, kudos, the authors' names, etc...) are cached properties. That means that once you check them once, the value is stored and it won't ever change, even if those values change. To update these values, you will need to call `Work.reload()`. See the example below:\n\n```python\nimport AO3\n\nsess = AO3.GuestSession()\nwork = AO3.Work(16721367, sess)\nprint(work.kudos)\nwork.leave_kudos()\nwork.reload()\nprint(work.kudos)\n```\n\n```\n392\n393\n```\n\n\n\n## Users\n\nAnother useful thing you might want to do is get information on who wrote which works / comments. For that, we use the `AO3.User` class.\n\n```python\nimport AO3\n\nuser = AO3.User(\"bothersomepotato\")\nprint(user.url)\nprint(user.bio)\nprint(user.works) # Number of works published\n```\n\n```\nhttps://archiveofourown.org/users/bothersomepotato\nUniversity student, opening documents to write essays but writing this stuff instead. No regrets though. My Tumblr, come chat with -or yell at- me if you feel like it! :)\n2\n```\n\n\n## Search\n\nTo search for works, you can either use the `AO3.search()` function and parse the BeautifulSoup object returned yourself, or use the `AO3.Search` class to automatically do that for you\n\n```python\nimport AO3\nsearch = AO3.Search(any_field=\"Clarke Lexa\", word_count=AO3.utils.Constraint(5000, 15000))\nsearch.update()\nprint(search.total_results)\nfor result in search.results:\n print(result)\n```\n\n```\n3074\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n```\n\nYou can then use the workid to load one of the works you searched for. To get more then the first 20 works, change the page number using \n```python\nsearch.page = 2\n```\n\n## Session\n\nA lot of actions you might want to take might require an AO3 account, and if you have one, you can get access to those actions using an AO3.Session object. You start by logging in using your username and password, and then you can use that object to access restricted content.\n\n```python\nimport AO3\n\nsession = AO3.Session(\"username\", \"password\")\nprint(f\"Bookmarks: {session.bookmarks}\")\nsession.refresh_auth_token()\nprint(session.kudos(AO3.Work(18001499, load=False))\n```\n\n```\nBookmarks: 67\nTrue\n```\n\nWe successfully left kudos in a work and checked our bookmarks. The `session.refresh_auth_token()` is needed for some activities such as leaving kudos and comments. If it is expired or you forget to call this function, the error `AO3.utils.AuthError: Invalid authentication token. Try calling session.refresh_auth_token()` will be raised.\n\nYou can also comment / leave kudos in a work by calling `Work.leave_kudos()`/`Work.comment()` and provided you have instantiated that object with a session already (`AO3.Work(xxxxxx, session=sess)` or using `Work.set_session()`). This is probably the best way to do so because you will run into less authentication issues (as the work's authenticity token will be used instead).\n\nIf you would prefer to leave a comment or kudos anonimously, you can use an `AO3.GuestSession` in the same way you'd use a normal session, except you won't be able to check your bookmarks, subscriptions, etc... because you're not actually logged in.\n\n\n## Comments\n\nTo retrieve and process comment threads, you might want to look at the `Work.get_comments()` method. It returns all the comments in a specific chapter and their respective threads. You can then process them however you want. Let's take a look:\n\n```python\nfrom time import time\n\nimport AO3\n\n\nwork = AO3.Work(24560008)\nwork.load_chapters()\nstart = time()\ncomments = work.get_comments(5)\nprint(f\"Loaded {len(comments)} comment threads in {round(time()-start, 1)} seconds\\n\")\nfor comment in comments:\n print(f\"Comment ID: {comment.id}\\nReplies: {len(comment.get_thread())}\")\n```\n\n```\nLoaded 5 comment threads in 1.8 seconds\n\nComment ID: 312237184\nReplies: 1\nComment ID: 312245032\nReplies: 1\nComment ID: 312257098\nReplies: 1\nComment ID: 312257860\nReplies: 1\nComment ID: 312285673\nReplies: 2\n```\n\nLoading comments takes a very long time so you should try and use it as little as possible. It also causes lots of requests to be sent to the AO3 servers, which might result in getting the error `utils.HTTPError: We are being rate-limited. Try again in a while or reduce the number of requests`. If it happens, you should try to space out your requests or reduce their number. There is also the option to enable request limiting using `AO3.utils.limit_requests()`, which make it so you can't make more than x requests in a certain time window.\nYou can also reply to comments using the `Comment.reply()` function, or delete one (if it's yours) using `Comment.delete()`.\n\n\n## Extra\n\nAO3.extra contains the the code to download some extra resources that are not core to the functionality of this package and don't change very often. One example would be the list of fandoms recognized by AO3.\nTo download a resource, simply use `AO3.extra.download(resource_name)`. To download every resource, you can use `AO3.extra.download_all()`. To see the list of available resources, `AO3.extra.get_resources()` will help you.\n" }, { "path": "mkdocs.yml", "content": "site_name: AO3 API\ndocs_dir: docs\ntheme: readthedocs\n\nnav:\n - Home: index.md\n - Installation: install.md\n - Usage: use.md" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"ao3-api\"\nversion = \"2.3.1\"\nauthors = [\n { name=\"Wendy\" },\n]\ndescription = \"An unofficial AO3 (archiveofourown.org) API\"\nreadme = \"README.md\"\nrequires-python = \">=3.8\"\nclassifiers = [\n \"Programming Language :: Python :: 3\",\n \"License :: OSI Approved :: MIT License\",\n \"Operating System :: OS Independent\",\n]\nkeywords = [\"ao3\", \"fanfiction\", \"Archive of Our Own\"]\n\ndependencies = [\n \"BeautifulSoup4\",\n \"lxml\",\n \"requests\"\n]\n\n[project.urls]\nHomepage = \"https://github.com/wendytg/ao3_api\"\nIssues = \"https://github.com/wendytg/ao3_api/issues\"\nDocumentation = \"https://ao3-api.readthedocs.io/\"\n" } ]