]{0,}[ ]{0,}((chapters|chapter) ([ivxl|0-9+]{1,}[,|-|–]){1,}[ivxl|0-9+]{1,})(.*$)'\n\n if re.search(pat2, summary, re.IGNORECASE):\n prefix = re.match(pat2, summary, re.IGNORECASE).group(1)\n summary = summary.replace(prefix,\"\")\n\n return summary.strip()\n\nmultiple_summaries = 0\ntotal_new_summaries = 0\nsummary_path_missing = []\ncounter = 0\n\n\nfp = open(matched_books, \"r\")\nfp_lines = fp.readlines()\n\nfor line in tqdm(fp_lines):\n line = line.strip()\n x = json.loads(line)\n counter += 1\n \n summaries_counted = 0\n\n prev_book_unique_id = \"\"\n\n book_unique_id = x['book_unique_id']\n\n source = basename(dirname(dirname(x['summary_path'])))\n book_name = basename(dirname(x['summary_path']))\n \n section_name = basename(x['summary_path'])\n\n # Capture the number of splits required from each summary file\n splits_reqd = [item['summary_path'] for item in x['splits']]\n num_splits_reqd = len(splits_reqd)\n \n if (book_unique_id != prev_book_unique_id) and prev_book_unique_id != \"\":\n summaries_counted = 0\n\n summary_path = os.path.join(\"../\", x['summary_path'])\n\n if not os.path.exists(summary_path):\n # Summary path missing\n summary_path_missing.append(summary_path)\n continue\n\n try:\n fx = open(summary_path, \"r\")\n except Exception as e:\n print (e)\n f_all_errors.write(\"Error loading summary path\" + \"\\t\" +summary_path)\n f_all_errors.write(\"\\n\")\n continue\n\n summary_json = json.load(fx)\n \n summary_content = summary_json['summary']\n\n og_summary_content = summary_content\n\n # Handling typos in specific books that can make splitting summaries easier\n if source == 'cliffnotes' and book_name == \"The Merry Wives of Windsor\" and 'section_10_part_0.txt' in x['summary_path']:\n summary_content = \"Scene 2 \" + summary_content\n\n if \"CHAPTER 1Summary\" in summary_content:\n summary_content = summary_content.replace(\"CHAPTER SUMMARIES WITH NOTES PHASE THE FIRST - THE MAIDEN\", \"\").replace(\"CHAPTER 1Summary\", \"CHAPTER 1 Summary\")\n\n if source == 'sparknotes' and book_name == \"Cyrano de Bergerac\":\n summary_content = summary_content.replace(\"Act IV, scenes vi-x Summary -- Act IV\", \"Summary -- Act IV\")\n\n if source == 'novelguide' and book_name == \"Henry VI Part 1\" and section_name == 'section_4_part_0.txt':\n summary_content = summary_content.replace('scence 6', 'scene 6')\n\n if source == 'novelguide' and book_name == \"Cyrano de Bergerac\" and section_name == 'section_8_part_0.txt':\n summary_content = summary_content.replace('Act, 5, scene 5', 'Act 5, scene 5')\n\n if source == 'pinkmonkey' and book_name == 'Emma' and section_name == 'section_10_part_0.txt':\n summary_content = summary_content.replace('CHAPTERS 11 & 12', 'CHAPTER 11-12')\n \n if source == 'pinkmonkey' and book_name == 'Jude the Obscure' and section_name == 'section_4_part_0.txt': \n summary_content = summary_content.replace('PART II CHAPTER 1 Summary', 'CHAPTER 1 Summary')\n\n summary_content = remove_prefixes_summary(summary_content)\n\n section_name_prefix = x['section_name_prefix']\n\n if '-' in section_name_prefix or section_name_prefix == 'epilogue':\n section_name_prefix = \"\"\n\n separated_summaries = separate_mulitple_summaries(summary_content, section_name_prefix, summary_path)\n\n total_new_summaries += len(separated_summaries.keys())\n\n print (\"num_splits_reqd: \", splits_reqd, num_splits_reqd)\n print (\"separated_summaries keys: \", separated_summaries.keys(), len(separated_summaries.keys()))\n \n assert num_splits_reqd <= len(separated_summaries.keys())\n # We can potentially split a summary text into more number of sections than we're able to match with the book chapters in gutenberg\n\n # No need to separate and use the new section name if we only have one single summary found after breaking\n # Also protects against some false positives wrt splitting the sections\n if len(separated_summaries.keys()) == 1:\n continue\n else:\n multiple_summaries += 1\n\n # If there is an empty section name, it means we couldn't find a section name to match that summary to a chapter\n if '' in separated_summaries.keys():\n separated_summaries.pop('')\n summaries_counted += save_separated_summaries(separated_summaries, summary_json, summary_path, book_name)\n else:\n summaries_counted += save_separated_summaries(separated_summaries, summary_json, summary_path, book_name)\n \n os.remove(summary_path)\n \n"
},
{
"path": "scripts/data_collection/bookwolf/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\n\n\"\"\"\nNote: Summaries collected through bookwolf require significant manual cleanup owing to the way the HTML is written\n\"\"\"\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom unidecode import unidecode\nimport time\nimport pdb\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/bookwolf/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210120012015/http://www.bookwolf.com/'\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\n#File for capturing the HTTP Errors, for webpages that are not found\nf_errors = open(\"section_errors.txt\",\"w\")\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\ndef get_overview_paragraphs(overview_links, specific_summary_dir):\n\n for index, (overview, name) in enumerate(overview_links):\n\n print (name, overview)\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n overview_data = soup.find(\"td\", {\"class\": \"TextObject\"})\n\n overview_paragraphs = [unidecode(paragraph.text.strip()) for paragraph in overview_data.findAll(\"p\", recursive=False)[1:]]\n \n except Exception as e:\n print(e)\n time.sleep(5)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n overview_data = soup.find(\"td\", {\"class\": \"TextObject\"})\n\n overview_paragraphs = [unidecode(paragraph.text.strip()) for paragraph in overview_data.findAll(\"p\", recursive=False)[1:]]\n\n except Exception as e:\n print(\"No book summary for: \", overview, e)\n f_errors.write(overview + \"\\t\" + name + \"\\t\" + specific_summary_dir + \"\\n\")\n continue\n\n overview_text = \"\\n\".join(overview_paragraphs)\n \n overview_dict = {}\n overview_dict[\"name\"] = \"Overview\"\n overview_dict[\"summary\"] = overview_text\n\n output_fname = os.path.join(specific_summary_dir, \"overview.json\")\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(overview_dict, fp)\n\ndef get_section_paragraphs(section_links, specific_summary_dir):\n #Fetch chapter level summary\n for index, (section, name) in enumerate(section_links):\n\n try:\n print (name, section)\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n section_data = soup.find(\"td\", {\"class\": \"TextObject\"})\n except Exception as e:\n\n print (e)\n time.sleep(5)\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n section_data = soup.find(\"td\", {\"class\": \"TextObject\"})\n except Exception as e:\n print (\"Chapter level summary not found for: \", section, e)\n f_errors.write(section + \"\\t\" + name + \"\\t\" + specific_summary_dir + \"\\n\")\n continue\n\n\n section_paragraphs = []\n section_analysis = []\n\n # Recursive function to parse HTML with upper case tags\n def parse_upper_case_html_tags(paragraph):\n\n paragraph_text = paragraph.text\n\n if \"\".join(section_paragraphs)\n\n section_dict = {}\n section_dict[\"name\"] = name\n section_dict[\"summary\"] = section_text\n\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % (index-1))\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)\n\n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n\n print('\\n>>> {}. {} <<<'.format(k, title))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory.\")\n # continue\n\n # Parse page\n print (\"page_url: \", page_url)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n f_errors.write(str(k) + \"\\t\" + title + \"\\t\" + page_url + \"\\t\" + specific_summary_dir + \"\\n\")\n continue\n\n\n # Parse general summary\n navigation_links = soup.find(\"table\", {\"id\": \"Table56\"})\n if navigation_links == None:\n navigation_links = soup.find(\"td\", {\"class\": \"TextObject\"})\n overview_links = [(urllib.parse.urljoin(MAIN_SITE, link.get(\"href\")), link.text) for link in navigation_links.findAll(\"a\")\\\n if (\"part\" not in link.text.lower() and (\"context\" in link.get(\"href\") or \"summary\" in link.get(\"href\") or \"synopsis\" in link.get(\"href\") ))]\n\n # Filter out some of the links that are obviously not chapter summary links\n # Since this source only has a handful of books, it was easy to hard code which links to fetch summaries from\n section_links = [(urllib.parse.urljoin(MAIN_SITE, link.get(\"href\")), link.text) for link in navigation_links.findAll(\"a\") \\\n if (\"interpretation\" not in link.text.lower() and \"comment\" not in link.text.lower() and \"author\" not in link.text.lower()\\\n and \"character\" not in link.text.lower() and \"questions\" not in link.text.lower() and \"life at the time\" not in link.text.lower()\\\n and \"theme\" not in link.text.lower() and \"foreword\" not in link.text.lower() and \"background\" not in link.text.lower()\\\n and \"symbolism\" not in link.text.lower() and \"introduction\" not in link.text.lower() and \"characterization\" not in link.text.lower()\\\n and \"setting\" not in link.text.lower() and \"family life\" not in link.text.lower() and \"comment\" not in link.text.lower() \\\n and \"context\" not in link.text.lower() ) ]\n \n if len(overview_links) != 0:\n get_overview_paragraphs(overview_links, specific_summary_dir)\n\n if len(section_links) != 0:\n get_section_paragraphs(section_links, specific_summary_dir)\n\n"
},
{
"path": "scripts/data_collection/bookwolf/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\n\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/'\nSEED_URL = 'https://web.archive.org/web/20210120012015/https://www.bookwolf.com/Welcome_to_Bookwolf1/welcome_to_bookwolf1.html'\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n scraped_links = []\n\n soup = BeautifulSoup(urllib.request.urlopen(seed_page), \"html.parser\")\n items = soup.findAll(\"tr\", {\"align\": \"LEFT\", \"valign\": \"TOP\"})\n tables = items[0].findAll(\"table\")\n books_table = tables[10].findAll(\"p\")\n print(\"Found %d items.\" % len(books_table))\n\n # # Go over each section\n for index, item in enumerate(books_table):\n # Parse section to get bullet point text\n item_title = item.find(\"a\").text\n item_url = item.find(\"a\").get(\"href\")\n\n print (index)\n print (\"item_title: \", item_title)\n print (\"item_url: \", item_url, \"\\n\")\n\n scraped_links.append({\n \"title\": item_title.strip().replace(\",\",\"\"),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/bookwolf/literature_links.tsv.pruned",
"content": "King Lear\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/King_Lear_free_booknotes/king_lear_free_booknotes.html\nJane Eyre\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Jane_Eyre_by_Charlotte_Bronte/jane_eyre.html\nMacbeth\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Macbeth/macbeth.html\nA Tale of Two Cities\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/A_Tale_of_Two_Cities/a_tale_of_two_cities.html\nPride and Prejudice\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Pride___Prejudice/pride___prejudice.html\nWuthering Heights\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Wuthering_Heights/wuthering_heights.html\nFrankenstein\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Frankenstein_by_Mary_Shelley/frankenstein_by_mary_shelley.html\nHamlet\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Hamlet/hamlet.html\nOthello\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Othello/othello.html\nJulius Caesar\thttps://web.archive.org/web/20210226213001/http://www.bookwolf.com/Free_Booknotes/Julius_Caesar/julius_caesar.html\n"
},
{
"path": "scripts/data_collection/cliffnotes/get_summaries.py",
"content": "\n\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom urllib.error import HTTPError, URLError\nimport time\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/cliffnotes/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210312193150/https://www.cliffsnotes.com/'\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\nerrors_file = open(\"section_errors.txt\",\"w\")\n\ndef wrap_data(name, summary, analysis, url):\n return {\n \"name\": name,\n \"summary\": summary,\n \"analysis\": analysis,\n \"url\": url\n }\n\n\ndef scrape_section_continuation(parent_soup, section_header):\n section_data = parent_soup.find(\"article\", {\"class\": \"copy\"})\n\n # For some links, the html structure is different\n if section_data == None:\n section_data = parent_soup.find(\"div\", {\"class\": \"contentArea\"})\n link = parent_soup.findAll(\"a\", {\"class\": \"cf-next icon-Next_Arrow\"}, href=True)[-1]\n next_link_title = link.findAll(\"p\")[-1].text.strip()\n else:\n link = parent_soup.findAll(\"a\", {\"class\": \"nav-bttn-filled\"}, href=True)[-1]\n next_link_title = link.findAll(\"span\")[-1].text.strip()\n\n section_paragraphs = [paragraph.text.strip() for paragraph in section_data.findAll(\"p\", recursive=False)]\n \n if not section_header == next_link_title:\n return section_paragraphs\n else:\n soup = BeautifulSoup(urllib.request.urlopen(urllib.parse.urljoin(MAIN_SITE, link.get(\"href\"))), \"html.parser\")\n return section_paragraphs + scrape_section_continuation(soup, section_header)\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as csvfile:\n reader = csv.reader(csvfile, delimiter='\\t')\n summary_infos = list(reader)\n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n print('\\n>>> {}. {} <<<'.format(k, title))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory, skipping.\")\n # continue\n\n # Parse page\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n errors_file.write(page_url + \"\\t\" + str(e) + \"\\n\")\n continue \n\n # Parse general summary\n navigation_links = soup.find(\"section\", {\"class\": \"secondary-navigation\"})\n overview_links = [urllib.parse.urljoin(MAIN_SITE, link.get(\"href\")) for link in navigation_links.findAll(\"a\") if re.match(\".*book-summary$\", link.get(\"href\"))]\n section_links = [urllib.parse.urljoin(MAIN_SITE, link.get(\"href\")) for link in navigation_links.findAll(\"a\") if \"summary-and-analysis\" in link.get(\"href\")][1:]\n\n for index, overview in enumerate(overview_links):\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n overview_data = soup.find(\"article\", {\"class\": \"copy\"})\n overview_paragraphs = filter(None, [paragraph.text.strip() for paragraph in overview_data.findAll(\"p\", recursive=False)])\n overview_text = \"\".join(overview_paragraphs).replace(\"Continued on next page...\", \"\")\n overview_data = wrap_data(\"Overview\", overview_text, None, overview)\n\n output_fname = os.path.join(specific_summary_dir, 'overview.txt')\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(overview_data))\n except Exception:\n print(\"No book summary\")\n\n \n for index, section in enumerate(section_links):\n \n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except Exception as e:\n print (section, e)\n errors_file.write(section + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n\n except Exception as e:\n print (section, e)\n errors_file.write(section + \"\\t\" + str(e) + \"\\n\")\n continue\n\n section_header = soup.title.string.strip()\n\n print (section_header, section)\n\n section_paragraphs = list(filter(None, scrape_section_continuation(soup, section_header)))\n section_text = \"\".join(section_paragraphs).replace(\"Continued on next page...\", \"\")\n\n # clean up and parse\n if \"Summary\\n\" in section_text and \"Analysis\\n\" in section_text:\n section_text_split = section_text.split(\"Analysis\\n\")\n summary_text = \"\".join([summary for summary in section_text_split if \"Summary\\n\" in summary]).replace(\"Summary\\n\", \"\").strip()\n analysis_text = \"\".join([analysis for analysis in section_text_split if \"Summary\\n\" not in analysis]).replace(\"Analysis\\n\", \"\").strip()\n else:\n summary_text = section_text.replace(\"Summary\\n\", \"\").strip()\n analysis_text = None\n\n section_data = wrap_data(section_header, summary_text, analysis_text, section)\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % index)\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(section_data))\n"
},
{
"path": "scripts/data_collection/cliffnotes/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210312193150/https://www.cliffsnotes.com/'\nSEED_URL = 'https://web.archive.org/web/20210312193150/https://www.cliffsnotes.com/literature?filter=ShowAll&sort=TITLE'\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n scraped_links = []\n\n soup = BeautifulSoup(urllib.request.urlopen(seed_page), \"html.parser\")\n items = soup.findAll(\"li\", {\"class\": \"note\"})\n print(\"Found %d items.\" % len(items))\n\n # Go over each section\n for index, item in enumerate(items):\n # Parse section to get bullet point text\n item_title = item.find(\"div\", {\"class\": \"note-name\"}).text\n item_url = item.find(\"a\").get(\"href\")\n\n scraped_links.append({\n \"title\": item_title.strip().replace(\",\",\"\"),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/cliffnotes/literature_links.tsv.pruned",
"content": "The Taming of the Shrew\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/the-taming-of-the-shrew/play-summary\nThe Prince\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/the-prince/book-summary\nSense and Sensibility\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/s/sense-and-sensibility/book-summary\nFar from the Madding Crowd\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/f/far-from-the-madding-crowd/book-summary\nTess of the d'Urbervilles\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/tess-of-the-durbervilles/book-summary\nThe Brothers Karamazov\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/b/the-brothers-karamazov/book-summary\nThe Picture of Dorian Gray\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/the-picture-of-dorian-gray/book-summary\nLord Jim\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/l/lord-jim/book-summary\nThe Red and the Black\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/the-red-and-the-black/book-summary\nThe Tempest\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/the-tempest/play-summary\nThe Portrait of a Lady\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/the-portrait-of-a-lady/book-summary\nThe Last of the Mohicans\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/l/the-last-of-the-mohicans/book-summary\nDracula\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/dracula/book-summary\nJane Eyre\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/j/jane-eyre/jane-eyre-at-a-glance\nAdam Bede\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/adam-bede/book-summary\nMadame Bovary\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/madame-bovary/book-summary\nMain Street\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/main-street/book-summary\nThe Deerslayer\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/the-deerslayer/book-summary\nOliver Twist\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/o/oliver-twist/book-summary\nThe House of the Seven Gables\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/the-house-of-the-seven-gables/book-summary\nVanity Fair\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/v/vanity-fair/book-summary\nWinesburg Ohio\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/w/winesburg-ohio/about-winesburg-ohio\nBabbitt\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/b/babbitt/sinclair-lewis-biography\nA Tale of Two Cities\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/a-tale-of-two-cities/a-tale-of-two-cities-at-a-glance\nThe House of Mirth\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/the-house-of-mirth/book-summary\nThe Scarlet Letter\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/s/the-scarlet-letter/the-scarlet-letter-at-a-glance\nAlice's Adventures in Wonderland\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/alices-adventures-in-wonderland/book-summary\nPride and Prejudice\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/pride-and-prejudice/book-summary\nGulliver's Travels\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/g/gullivers-travels/book-summary\nWuthering Heights\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/w/wuthering-heights/wuthering-heights-at-a-glance\nFrankenstein\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/f/frankenstein/frankenstein-at-a-glance\nJude the Obscure\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/j/jude-the-obscure/book-summary\nSiddhartha\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/s/siddhartha/about-siddhartha\nIvanhoe\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/i/ivanhoe/book-summary\nTreasure Island\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/treasure-island/book-summary\nDavid Copperfield\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/david-copperfield/book-summary\nSister Carrie\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/s/sister-carrie/book-summary\nDr. Jekyll and Mr. Hyde\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/dr-jekyll-and-mr-hyde/book-summary\nThe Turn of the Screw\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/the-turn-of-the-screw/book-summary\nCandide\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/c/candide/book-summary\nParadise Lost\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/paradise-lost/poem-summary\nMy Antonia\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/my-aacutentonia/book-summary\nHamlet\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/hamlet/hamlet-at-a-glance\nA Midsummer Night's Dream\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/a-midsummer-nights-dream/play-summary\nOthello\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/o/othello/play-summary\nKing Henry IV Part 1\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/k/king-henry-iv-part-1/play-summary\nNarrative of the Life of Frederick Douglass: An American Slave\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/n/narrative-of-the-life-of-frederick-douglass-an-american-slave/book-summary\nHeart of Darkness\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/heart-of-darkness/heart-of-darkness-at-a-glance\nGreen Mansions\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/g/green-mansions/book-summary\nThe Power and the Glory\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/the-power-and-the-glory/about-the-power-and-the-glory\nThe Return of the Native\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/the-return-of-the-native/book-summary\nThe Pickwick Papers\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/the-pickwick-papers/book-summary\nThe Rise of Silas Lapham\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/the-rise-of-silas-lapham/about-the-rise-of-silas-lapham\nThe Red Badge of Courage\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/the-red-badge-of-courage/book-summary\nThe Mill on the Floss\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/the-mill-on-the-floss/book-summary\nThe Way of All Flesh\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/w/the-way-of-all-flesh/book-summary\nIncidents in the Life of a Slave Girl\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/i/incidents-in-the-life-of-a-slave-girl/about-incidents-in-the-life-of-a-slave-girl\nMacbeth\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/macbeth/macbeth-at-a-glance\nRichard II\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/richard-ii/play-summary\nRichard III\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/richard-iii/play-summary\nHenry V\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/henry-v/play-summary\nJulius Caesar\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/j/julius-caesar/play-summary\nLove's Labour's Lost\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/l/loves-labours-lost/play-summary\nPygmalion\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/pygmalion/play-summary\nTwelfth Night\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/twelfth-night/play-summary\nTwo Gentlemen of Verona\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/two-gentlemen-of-verona/play-summary\nA Doll's House\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/a-dolls-house/play-summary\nArms and the Man\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/arms-and-the-man/play-summary\nGhosts\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/g/ghosts/play-summary\nHedda Gabler\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/h/hedda-gabler/play-summary\nThe Merry Wives of Windsor\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/the-merry-wives-of-windsor/play-summary\nDon Juan\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/d/don-juan/poem-summary\nTartuffe\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/tartuffe/play-summary\nAn Enemy of the People\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/e/an-enemy-of-the-people/play-summary\nThe Secret Sharer\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/s/the-secret-sharer/story-summary\nMan and Superman\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/man-and-superman/about-man-and-superman\nIdylls of the King\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/i/idylls-of-the-king/about-idylls-of-the-king\nRomeo and Juliet\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/r/romeo-and-juliet/romeo-and-juliet-at-a-glance\nPhaedra\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/p/phaedra/play-summary\nThe Merchant of Venice\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/the-merchant-of-venice/play-summary\nAll's Well That Ends Well\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/alls-well-that-ends-well/play-summary\nMajor Barbara\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/major-barbara/play-summary\nThe Three Musketeers\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/t/the-three-musketeers/book-summary\nCyrano de Bergerac\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/c/cyrano-de-bergerac/play-summary\nThe Winter's Tale\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/w/the-winters-tale/play-summary\nKing Lear\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/k/king-lear/play-summary\nBleak House\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/b/bleak-house/book-summary\nThe Adventures of Huckleberry Finn\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/the-adventures-of-huckleberry-finn/the-adventures-of-huckleberry-finn-at-a-glance\nAnthem\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/anthem/book-summary\nEthan Frome\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/e/ethan-frome/book-summary\nThe Jungle\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/j/the-jungle/about-the-jungle\nThe Ambassadors\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/the-ambassadors/book-summary\nThe Age of Innocence\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/the-age-of-innocence/book-summary\nThe Importance of Being Earnest\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/i/the-importance-of-being-earnest/play-summary\nThe Education of Henry Adams\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/e/the-education-of-henry-adams/book-summary\nMuch Ado About Nothing\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/m/much-ado-about-nothing/play-summary\nAntony and Cleopatra\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/antony-and-cleopatra/play-summary\nAs You Like It\thttps://web.archive.org/web/20201107055349/https://www.cliffsnotes.com/literature/a/as-you-like-it/play-summary\n"
},
{
"path": "scripts/data_collection/gradesaver/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\nfrom urllib.error import HTTPError, URLError\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/gradesaver/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210630200124/https://www.gradesaver.com/'\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\nerrors_file = open(\"section_errors.txt\",\"w\")\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n print('\\n>>> {}. {} {} <<<'.format(k, title, page_url))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory, skipping.\")\n # continue\n\n # Parse page\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except URLError as err:\n print (err, page_url, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n errors_file.write(page_url + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n except Exception as e:\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n errors_file.write(page_url + \"\\t\" + str(e))\n continue\n\n\n # # Parse general summary\n navigation_links = soup.find(\"ul\", {\"class\": \"navSection__list js--collapsible\"})\n overview_links = [(urllib.parse.urljoin(MAIN_SITE, link.find(\"a\").get(\"href\")), link.text.strip()) for link in navigation_links.findAll(\"li\") if link.text.strip() == title + \" Summary\"]\n print (\"overview_links: \", overview_links)\n\n if len(overview_links) == 0:\n print (\"No overview summaries found\")\n else:\n for index, (overview, name) in enumerate(overview_links):\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n overview_data = soup.find(\"article\", {\"class\": \"section__article\"})\n\n overview_paragraphs = []\n overview_analysis = []\n start = 1\n for paragraph in overview_data.findAll(\"p\", recursive=False):\n\n #Skip the first word if it is \"Summary\" or \"Context\"\n if paragraph.text.strip().lower() in [\"summary\", \"context\"]:\n continue\n\n if paragraph.text.strip().lower() in [\"analysis\"]:\n start = 0\n continue\n \n if start:\n overview_paragraphs.append(paragraph.text.strip())\n else:\n overview_analysis.append(paragraph.text.strip())\n \n overview_text = \"\".join(overview_paragraphs)\n overview_analysis_text = \"\".join(overview_analysis)\n \n overview_dict = {}\n overview_dict[\"name\"] = \"Overview\"\n overview_dict[\"summary\"] = overview_text\n overview_dict[\"analysis\"] = overview_analysis_text\n overview_dict[\"url\"] = overview\n \n\n # print (overview_analysis_text)\n output_fname = os.path.join(specific_summary_dir, \"overview.json\")\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(overview_dict, fp)\n\n except Exception as e:\n print (\"No overview summary for: \", overview)\n print (e)\n\n section_links = [link.find(\"ul\").findAll(\"li\") for link in navigation_links.findAll(\"li\") if \"Summary And Analysis\" in link.text.strip()]\n \n if len(section_links) == 0:\n print (\"No section summaries found\")\n else:\n section_links = [(urllib.parse.urljoin(MAIN_SITE,link.find(\"a\").get(\"href\")), link.text.strip()) for link in section_links[0]]\n\n for index, (section, name) in enumerate(section_links):\n try:\n print (name, section)\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except URLError as err:\n print (err, section, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except Exception as e:\n print (section, e)\n errors_file.write(section + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n \n except Exception as e:\n print (\"No section summary found for: \", section, e)\n errors_file.write(section + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n\n section_data = soup.find(\"article\", {\"class\": \"section__article\"})\n section_paragraphs = []\n section_analysis = []\n start = 1\n \n for paragraph in section_data.findAll([\"p\",\"h2\", \"h3\", \"h4\"]):\n \n # Handle chapter name occuring before the title 'Summary'\n # We want to capture it, not skip it\n if re.search('^(chapter [ivxl|0-9]+)', paragraph.text.strip().lower(), re.IGNORECASE):\n section_paragraphs.append(paragraph.text.strip())\n continue\n\n #Skip the first sentence if it is \"Summary\" or \"Context\"\n elif paragraph.text.strip().lower() in [\"summary\", \"context\"]:\n start = 1\n continue\n\n if paragraph.text.strip().lower() in [\"analysis\"]:\n start = 0\n continue\n \n if start:\n section_paragraphs.append(paragraph.text.strip())\n else:\n section_analysis.append(paragraph.text.strip())\n\n # print (\"section_paragraphs: \", section_paragraphs)\n \n section_text = \"\".join(section_paragraphs)\n section_analysis_text = \"\".join(section_analysis)\n\n # print (\"section_text: \", section_text)\n\n section_dict = {}\n section_dict[\"name\"] = name\n section_dict[\"summary\"] = section_text\n section_dict[\"analysis\"] = section_analysis_text\n section_dict[\"url\"] = section\n \n\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % index)\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)"
},
{
"path": "scripts/data_collection/gradesaver/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io, string\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210226083212/https://www.gradesaver.com/'\n\n\nSEED_URL = 'https://web.archive.org/web/20210226083212/https://www.gradesaver.com/study-guides/'\n\nalphabet_list = string.ascii_uppercase\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n\n scraped_links = []\n\n for char in alphabet_list:\n books_page = seed_page + char\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n except Exception as e:\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n except Exception as e:\n print (\"Skipping book: \", books_page)\n errors_file.write(books_page + \"\\t\" + str(e) + \"\\n\")\n\n items = soup.findAll(\"ul\", {\"class\": \"columnList\"})\n books = items[0].findAll(\"li\", {\"class\":\"columnList__item\"})\n\n # # Go over each section\n for index, item in enumerate(books):\n # Parse section to get bullet point text\n\n item_title = item.find(\"a\").text\n item_url = item.find(\"a\").get(\"href\")\n\n print (\"item_title: \", item_title.strip())\n print (\"item_url: \", item_url.strip(), \"\\n\")\n\n scraped_links.append({\n \"title\": item_title.strip(),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n \n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/gradesaver/literature_links.tsv.pruned",
"content": "The Prince\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/the-prince\nSense and Sensibility\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/sense-and-sensibility\nFar from the Madding Crowd\thttps://web.archive.org/web/20210417171037/https://www.gradesaver.com/far-from-the-madding-crowd\nTess of the D'Urbervilles\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/tess-of-the-durbervilles\nThe Brothers Karamazov\thttps://web.archive.org/web/20210417165227/https://www.gradesaver.com/the-brothers-karamazov\nThe Picture of Dorian Gray\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/the-picture-of-dorian-gray\nLord Jim\thttps://web.archive.org/web/20210417183139/https://www.gradesaver.com/lord-jim\nThe Consolation of Philosophy\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/the-consolation-of-philosophy\nThe Rime of the Ancient Mariner\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/the-rime-of-the-ancient-mariner\nThe Autobiography of an Ex-Colored Man\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/the-autobiography-of-an-excolored-man\nThe White Devil\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/the-white-devil\nThe Tempest\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/the-tempest\nKing Solomon's Mines\thttps://web.archive.org/web/20210417175853/https://www.gradesaver.com/king-solomons-mines\nGargantua and Pantagruel\thttps://web.archive.org/web/20210417171712/https://www.gradesaver.com/gargantua-and-pantagruel\nUncle Vanya\thttps://web.archive.org/web/20210417165119/https://www.gradesaver.com/uncle-vanya\nThe Portrait of a Lady\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/the-portrait-of-a-lady\nThe School for Scandal\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-school-for-scandal\nDracula\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/dracula\nJane Eyre\thttps://web.archive.org/web/20210417165212/https://www.gradesaver.com/jane-eyre\nAdam Bede\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/adam-bede\nThe Hound of the Baskervilles\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/the-hound-of-the-baskervilles\nThe Aeneid\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/the-aeneid\nOliver Twist\thttps://web.archive.org/web/20210417180116/https://www.gradesaver.com/oliver-twist\nThe House of the Seven Gables\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/the-house-of-the-seven-gables\nVanity Fair\thttps://web.archive.org/web/20210417172348/https://www.gradesaver.com/vanity-fair\nLittle Women\thttps://web.archive.org/web/20210417183139/https://www.gradesaver.com/little-women\nBabbitt\thttps://web.archive.org/web/20210417165227/https://www.gradesaver.com/babbitt\nA Tale of Two Cities\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/tale-of-two-cities\nThe Scarlet Letter\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-scarlet-letter\nEmma\thttps://web.archive.org/web/20210417182948/https://www.gradesaver.com/emma\nPersuasion\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/persuasion\nPride and Prejudice\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/pride-and-prejudice\nGulliver's Travels\thttps://web.archive.org/web/20210417171712/https://www.gradesaver.com/gullivers-travels\nWuthering Heights\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/wuthering-heights\nFrankenstein\thttps://web.archive.org/web/20210417171037/https://www.gradesaver.com/frankenstein\nMansfield Park\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/mansfield-park\nMiddlemarch\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/middlemarch\nSiddhartha\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/siddhartha\nTreasure Island\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/treasure-island\nDavid Copperfield\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/david-copperfield\nSister Carrie\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/sister-carrie\nKidnapped\thttps://web.archive.org/web/20210417175853/https://www.gradesaver.com/kidnapped\nA Christmas Carol\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/a-christmas-carol\nDr. Jekyll and Mr. Hyde\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/dr-jekyll-and-mr-hyde\nThe Turn of the Screw\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/turn-of-the-screw\nCandide\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/candide\nParadise Lost\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/paradise-lost\nThe Federalist Papers\thttps://web.archive.org/web/20210417171037/https://www.gradesaver.com/the-federalist-papers\nThe Rivals\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/the-rivals\nNorthanger Abbey\thttps://web.archive.org/web/20210417165319/https://www.gradesaver.com/northanger-abbey\nMy Antonia\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/my-antonia\nA Vindication of the Rights of Woman\thttps://web.archive.org/web/20210417172348/https://www.gradesaver.com/a-vindication-of-the-rights-of-woman\nA Study in Scarlet\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/a-study-in-scarlet\nHamlet\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/hamlet\nA Midsummer Night's Dream\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/midsummer-nights-dream\nOthello\thttps://web.archive.org/web/20210417180116/https://www.gradesaver.com/othello\nA Room With a View\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/a-room-with-a-view\nMary Barton\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/mary-barton\nCoriolanus\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/coriolanus\nThe Garden Party\thttps://web.archive.org/web/20210417171712/https://www.gradesaver.com/the-garden-party\nAround the World in 80 Days\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/around-the-world-in-80-days\nWhat Maisie Knew\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/what-maisie-knew\nThe Vicar of Wakefield\thttps://web.archive.org/web/20210417172348/https://www.gradesaver.com/the-vicar-of-wakefield\nUtilitarianism\thttps://web.archive.org/web/20210417165119/https://www.gradesaver.com/utilitarianism\nHeart of Darkness\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/heart-of-darkness\nA Hero of Our Time\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/a-hero-of-our-time\nThe Phantom of the Opera\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/the-phantom-of-the-opera\nLeviathan\thttps://web.archive.org/web/20210417183139/https://www.gradesaver.com/leviathan\nThe Canterville Ghost\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/the-canterville-ghost\nVillette\thttps://web.archive.org/web/20210417172348/https://www.gradesaver.com/villette\nOn Liberty\thttps://web.archive.org/web/20210417180116/https://www.gradesaver.com/on-liberty\nThe Marrow of Tradition\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/the-marrow-of-tradition\nThe Wind in the Willows\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/the-wind-in-the-willows\nHowards End\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/howards-end\nThe Secret Agent\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-secret-agent\nAnne of Green Gables\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/anne-of-green-gables\nA Little Princess\thttps://web.archive.org/web/20210417183139/https://www.gradesaver.com/a-little-princess\nRegeneration\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/regeneration\nTherese Raquin\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/therese-raquin\nThe Monkey's Paw\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/the-monkeys-paw\nTwelve Years a Slave\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/twelve-years-a-slave\nHouse of Mirth\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/house-of-mirth\nThe Rise of Silas Lapham\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/the-rise-of-silas-lapham\nThe Red Badge of Courage\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/the-red-badge-of-courage\nO Pioneers\thttps://web.archive.org/web/20210417180116/https://www.gradesaver.com/o-pioneers\nThe Mill on the Floss\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/the-mill-on-the-floss\nThe Time Machine\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/the-time-machine\nThe Valley of Fear\thttps://web.archive.org/web/20210417172348/https://www.gradesaver.com/the-valley-of-fear\nNews from Nowhere\thttps://web.archive.org/web/20210417165319/https://www.gradesaver.com/news-from-nowhere\nLittle Dorrit\thttps://web.archive.org/web/20210417183139/https://www.gradesaver.com/little-dorrit\nIncidents in the Life of a Slave Girl\thttps://web.archive.org/web/20210417175859/https://www.gradesaver.com/incidents-in-the-life-of-a-slave-girl\nHenry IV Part 2\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/henry-iv-part-2\nMacbeth\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/macbeth\nTroilus and Cressida\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/troilus-and-cressida\nRichard II\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/richard-ii\nHenry IV Part 1\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/henry-iv-part-1\nRichard III\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/richard-iii\nHenry V\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/henry-v\nEvery Man in His Humour\thttps://web.archive.org/web/20210417182948/https://www.gradesaver.com/every-man-in-his-humour\nTitus Andronicus\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/titus-andronicus\nJulius Caesar\thttps://web.archive.org/web/20210417165212/https://www.gradesaver.com/julius-caesar\nSecond Treatise of Government\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/second-treatise-of-government\nShe Stoops to Conquer\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/she-stoops-to-conquer\nThe Wonderful Wizard of Oz\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/the-wonderful-wizard-of-oz\nPygmalion\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/pygmalion\nTwelfth Night\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/twelfth-night\nA Doll's House\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/a-dolls-house\nArms and the Man\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/arms-and-the-man\nGhosts\thttps://web.archive.org/web/20210417171712/https://www.gradesaver.com/ghosts\nThe Spanish Tragedy\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-spanish-tragedy\nTartuffe\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/tartuffe\nAn Enemy of the People\thttps://web.archive.org/web/20210417182948/https://www.gradesaver.com/an-enemy-of-the-people\nMeasure for Measure\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/measure-for-measure\nThe Seagull\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-seagull\nAlice in Wonderland\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/alice-in-wonderland\nNjal's Saga\thttps://web.archive.org/web/20210417165319/https://www.gradesaver.com/njals-saga\nWhite Fang\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/white-fang\nRomeo and Juliet\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/romeo-and-juliet\nThe Ramayana\thttps://web.archive.org/web/20210417180420/https://www.gradesaver.com/the-ramayana\nCymbeline\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/cymbeline\nTroilus and Criseyde\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/troilus-and-criseyde\nThe Sorrows of Young Werther\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-sorrows-of-young-werther\nThe Hairy Ape\thttps://web.archive.org/web/20210417171128/https://www.gradesaver.com/the-hairy-ape\nAll for Love\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/all-for-love\nThe Jew of Malta\thttps://web.archive.org/web/20210417165212/https://www.gradesaver.com/the-jew-of-malta\nThe Coquette\thttps://web.archive.org/web/20210417175939/https://www.gradesaver.com/the-coquette\nAn Ideal Husband\thttps://web.archive.org/web/20210417175859/https://www.gradesaver.com/an-ideal-husband\nThe Three Musketeers\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/the-three-musketeers\nBlack Beauty\thttps://web.archive.org/web/20210417165227/https://www.gradesaver.com/black-beauty\nEvelina\thttps://web.archive.org/web/20210417182948/https://www.gradesaver.com/evelina-or-the-history-of-a-young-ladys-entrance-into-the-world\nThe Winter's Tale\thttps://web.archive.org/web/20210417183140/https://www.gradesaver.com/the-winters-tale\nKing Lear\thttps://web.archive.org/web/20210417175853/https://www.gradesaver.com/king-lear\nBleak House\thttps://web.archive.org/web/20210417165227/https://www.gradesaver.com/bleak-house\nThe Adventures of Huckleberry Finn\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/the-adventures-of-huckleberry-finn\nAnthem\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/anthem\nEthan Frome\thttps://web.archive.org/web/20210417182948/https://www.gradesaver.com/ethan-frome\nThe Jungle\thttps://web.archive.org/web/20210417165212/https://www.gradesaver.com/the-jungle\nThe Age of Innocence\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/the-age-of-innocence\nMrs. Warren's Profession\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/mrs-warrens-profession\nThe Blithedale Romance\thttps://web.archive.org/web/20210417165227/https://www.gradesaver.com/the-blithedale-romance\nDombey and Son\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/dombey-and-son\nMuch Ado About Nothing\thttps://web.archive.org/web/20210417171658/https://www.gradesaver.com/much-ado-about-nothing\nThe Secret Garden\thttps://web.archive.org/web/20210417183050/https://www.gradesaver.com/the-secret-garden\nThe Playboy of the Western World\thttps://web.archive.org/web/20210417180108/https://www.gradesaver.com/the-playboy-of-the-western-world\nThe Duchess of Malfi\thttps://web.archive.org/web/20210417171105/https://www.gradesaver.com/the-duchess-of-malfi\nThe Taming of the Shrew\thttps://web.archive.org/web/20210212015315/https://www.gradesaver.com/the-taming-of-the-shrew\nAntony and Cleopatra\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/antony-and-cleopatra\nAs You Like It\thttps://web.archive.org/web/20210417172435/https://www.gradesaver.com/as-you-like-it\n"
},
{
"path": "scripts/data_collection/novelguide/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json, time\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom unidecode import unidecode\nimport argparse\n\nPARSER = argparse.ArgumentParser(description='For processing HTTP errors separately')\nPARSER.add_argument(\"--fix_scraping_errors\", action=\"store_true\", help=\"Flag indicating \\\nthat script should recrape the links it missed\")\nARGS = PARSER.parse_args()\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/novelguide/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210708184822/https://www.novelguide.com/'\n\ndef hasNumbers(inputString):\n return any(char.isdigit() for char in inputString)\n\ndef get_section_level_data(section_links):\n\n http_errors = []\n\n for index, (section, name), specific_summary_dir in section_links:\n \n print (name, section)\n \n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n section_data = soup.find(\"div\", {\"class\": \"content clear-block\"})\n\n section_paragraphs = []\n section_analysis = []\n\n section_paras = section_data.findAll(\"p\")\n \n for para in section_paras:\n if para.text.strip():\n section_paragraphs.append(unidecode(para.text.strip()))\n\n # Try alternate\n if section_paragraphs == []:\n section_paras = section_data.findAll(\"div\")\n # print (\"section_paras: \", section_paras)\n for para in section_paras:\n section_paragraphs.append(unidecode(para.text.strip()))\n\n\n section_text = \"\".join(section_paragraphs)\n section_analysis_text = \"\".join(section_analysis)\n #Actual analysis text to be extracted later\n\n section_dict = {}\n\n # All section names have colons, so we can get the section name by splitting on colons and taking the last item\n section_dict[\"name\"] = unidecode(name.split(':')[-1].strip())\n section_dict[\"summary\"] = section_text\n section_dict[\"analysis\"] = section_analysis_text\n section_dict[\"url\"] = unidecode(section)\n \n #Check for the overview\n if section_dict[\"name\"] in [\"Overview\", \"Novel Summary\", \"NovelSummary\", \"Summary\"]:\n output_fname = os.path.join(specific_summary_dir, 'overview.txt')\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)\n \n else: #Must be section file\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % int(index))\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)\n\n except Exception as e:\n print (\"No section summary for: \", section)\n print (e)\n time.sleep(5)\n\n http_errors.append((index, section, name, specific_summary_dir))\n # print (\"http_errors: \", http_errors)\n\n #Errors File is created for saving urls that are not found, before calling this function\n f_errors = open(\"section_errors.txt\",\"a\")\n\n for (index, section, name, specific_summary_dir) in http_errors:\n f_errors.write(str(index) + \"\\t\" + section + \"\\t\" + name + \"\\t\" + specific_summary_dir + \"\\n\")\n\n\n# fetch only the links that resulted in an http error\nif ARGS.fix_scraping_errors:\n if not os.path.exists(\"section_errors.txt\"):\n print (\"No errors file found\\nRun without scraping errors flag\")\n exit()\n else:\n f_errors = open(\"section_errors.txt\",\"r\")\n section_links = []\n for line in f_errors:\n line_splits = line.rstrip().split(\"\\t\")\n section_links.append((line_splits[0],(line_splits[1], line_splits[2]), line_splits[3]))\n \n f_errors.close()\n\n if len(section_links) == 0:\n print (\"No errors found\\nRun without scraping errors flag\")\n exit()\n \n print (\"Links with scraping errors scraped again: \", section_links)\n\n \n #Create the errors file every time when starting to scrape the summaries\n #Should overwrite the same file we opened for reading\n f_errors = open(\"section_errors.txt\",\"w\")\n\n #fetch the summaries using the links that threw an error\n get_section_level_data(section_links)\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\n\n# Create the errors file every time when starting to scrape the summaries\n# This file can be used to try and rescrape the links that resulted in an error\nf_errors = open(\"section_errors.txt\",\"w\")\n\nf_book_errors = open(\"book_errors.txt\",\"w\")\n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n print('\\n>>> {}. {} <<<'.format(k, title))\n\n overview_found = 0\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory.\")\n # continue\n\n # Parse page\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except urllib.error.HTTPError:\n print (\"HTTP error raised. Trying again\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except urllib.error.HTTPError:\n print (\"Page not accessible: \", page_url)\n f_book_errors.write(str(k) + \"\\t\" + title + \"\\t\" + page_url)\n f_book_errors.write(\"\\n\")\n continue\n\n # # Parse general summary\n navigation_links = soup.find(\"div\", {\"id\": \"block-booknavigation-3\"})\n\n # Some links are just empty webpages\n if navigation_links == None:\n print (\"Navigation links not found\")\n f_book_errors.write(str(k) + \"\\t\" + title + \"\\t\" + page_url)\n f_book_errors.write(\"\\n\")\n continue\n\n section_links = [(urllib.parse.urljoin(MAIN_SITE, link.find(\"a\").get(\"href\")), link.text.strip()) for link in navigation_links.findAll(\"li\")\\\n if 'chapter' in link.text.strip().lower() or 'summary' in link.text.strip().lower() or 'section' in link.text.strip().lower() or 'stave' in link.text.strip().lower() \\\n or 'chp' in link.text.strip().lower() or 'scene' in link.text.strip().lower() or 'act ' in link.text.strip().lower() \\\n or 'part' in link.text.strip().lower() or 'pages' in link.text.strip().lower() or 'lines' in link.text.strip().lower() \\\n or 'book' in link.text.strip().lower() or hasNumbers(link.text.strip().lower()) or 'overview' in link.text.strip().lower()\\\n or 'prologue' in link.text.strip().lower() or 'epilogue' in link.text.strip().lower()]\n #Why not checking for the keyword 'summary'??\n\n # Append index to all the section links\n section_links_with_index = []\n for index, (section, name) in enumerate(section_links):\n section_links_with_index.append((index,(section, name), specific_summary_dir))\n\n if len(section_links_with_index) == 0:\n print (\"No section summaries found\")\n else:\n get_section_level_data(section_links_with_index)\n"
},
{
"path": "scripts/data_collection/novelguide/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io, string\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210225014436/https://www.novelguide.com/'\nSEED_URL = 'https://web.archive.org/web/20210225014436/https://www.novelguide.com/title/'\n\nalphabet_list = string.ascii_lowercase + '1'\n\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n\n scraped_links = []\n\n for char in alphabet_list:\n page_no = 1\n books_page = seed_page + char\n\n while(True):\n\n try:\n\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n items = soup.findAll(\"ul\", {\"class\": \"search-title\"})\n books = items[0].findAll(\"li\")\n\n # # Go over each section\n for index, item in enumerate(books):\n # Parse section to get bullet point text\n\n item_title = item.find(\"a\").text\n item_url = item.find(\"a\").get(\"href\")\n\n print (\"item_title: \", item_title.strip())\n print (\"item_url: \", item_url.strip())\n print (\"\\n\")\n\n scraped_links.append({\n \"title\": item_title.strip(),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n\n except Exception as e:\n print (books_page, str(e))\n errors_file.write(books_page + \"\\t\" + str(e) + \"\\n\")\n\n break\n\n books_page = seed_page + char + \"?page=\" + str(page_no)\n page_no += 1\n \n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/novelguide/literature_links.tsv.pruned",
"content": "Consolation of Philosophy\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/consolation-of-philosophy\nMansfield Park\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/mansfield-park/index.html\nMy Antonia\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/my-antonia/index.html\nUtopia\thttps://web.archive.org/web/20210214151736/https://www.novelguide.com/utopia\nMerry Wives of Windsor\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/merry-wives-of-windsor\nLord Jim\thttps://web.archive.org/web/20201204111055/https://www.novelguide.com/lord-jim/index.html\nCommon Sense\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/common-sense/introduction\nRichard II\thttps://web.archive.org/web/20210708184822/https://www.novelguide.com/richard-ii/index.html\nLove's Labours Lost\thttps://web.archive.org/web/20201204111055/https://www.novelguide.com/loves-labours-lost/index.html\nBleak House\thttps://web.archive.org/web/20201020010735/https://www.novelguide.com/bleak-house\nSense and Sensibility\thttps://web.archive.org/web/20210212220225/https://www.novelguide.com/sense-and-sensibility/index.html\nTess of the d'Urbervilles\thttps://web.archive.org/web/20210212221210/https://www.novelguide.com/tess-of-the-durbervilles\nDracula\thttps://web.archive.org/web/20210212220747/https://www.novelguide.com/dracula/index.htm\nJane Eyre\thttps://web.archive.org/web/20201022042445/https://www.novelguide.com/jane-eyre/index.html\nAdam Bede\thttps://web.archive.org/web/20210213005530/https://www.novelguide.com/adam-bede\nMadame Bovary\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/madame-bovary/index\nMain Street\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/main-street/index.html\nOliver Twist\thttps://web.archive.org/web/20201204103600/https://www.novelguide.com/oliver-twist/index.html\nVanity Fair\thttps://web.archive.org/web/20210212235930/https://www.novelguide.com/vanity-fair/index.html\nWinesburg, Ohio\thttps://web.archive.org/web/20210228134637/https://www.novelguide.com/winesburg-ohio/index.html\nLittle Women\thttps://web.archive.org/web/20201204111055/https://www.novelguide.com/little-women/index.html\nBabbitt\thttps://web.archive.org/web/20201020010735/https://www.novelguide.com/babbitt\nA Tale of Two Cities\thttps://web.archive.org/web/20210213005530/https://www.novelguide.com/a-tale-of-two-cities\nEmma\thttps://web.archive.org/web/20210215030716/https://www.novelguide.com/emma\nPersuasion\thttps://web.archive.org/web/20210211232247/https://www.novelguide.com/persuasion/index.html\nPride and Prejudice\thttps://web.archive.org/web/20210211232247/https://www.novelguide.com/pride-and-prejudice/index.html\nWuthering Heights\thttps://web.archive.org/web/20210228134637/https://www.novelguide.com/wuthering-heights/index.html\nFrankenstein\thttps://web.archive.org/web/20201103045617/https://www.novelguide.com/frankenstein\nMiddlemarch\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/middlemarch\nSiddhartha\thttps://web.archive.org/web/20210212220225/https://www.novelguide.com/siddhartha\nIvanhoe\thttps://web.archive.org/web/20201205184924/https://www.novelguide.com/ivanhoe\nDavid Copperfield\thttps://web.archive.org/web/20210212220747/https://www.novelguide.com/david-copperfield/index\nSister Carrie\thttps://web.archive.org/web/20210212220225/https://www.novelguide.com/sister-carrie/index.html\nKidnapped\thttps://web.archive.org/web/20210228115510/https://www.novelguide.com/kidnapped\nDr. Jekyll and Mr. Hyde\thttps://web.archive.org/web/20210212220747/https://www.novelguide.com/dr-jekyll-and-mr-hyde/index.html\nCandide\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/candide/index\nHamlet\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/hamlet\nA Midsummer Night's Dream\thttps://web.archive.org/web/20210213005530/https://www.novelguide.com/a-midsummer-nights-dream\nOthello\thttps://web.archive.org/web/20201204103600/https://www.novelguide.com/othello\nA Room With a View\thttps://web.archive.org/web/20210213005530/https://www.novelguide.com/a-room-with-a-view/index\nCoriolanus\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/coriolanus/index\nAround the World in Eighty Days\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/around-the-world-in-eighty-days\nHeart of Darkness\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/heart-of-darkness\nNotes from the Underground\thttps://web.archive.org/web/20201022042315/https://www.novelguide.com/notes-from-the-underground\nMaggie A Girl of the Streets\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/maggie-a-girl-of-the-streets\nHenry VIII\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/henry-viii/index.html\nHenry IV Part 2\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/henry-iv-part2\nMacbeth\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/macbeth\nHenry IV Part 1\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/henry-iv-part-1/index.html\nHenry VI Part 1\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/henry-vi-part-1/index.html\nRichard III\thttps://web.archive.org/web/20210213004944/https://www.novelguide.com/richard-iii/introduction\nHenry V\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/henry-v/index.html\nJulius Caesar\thttps://web.archive.org/web/20201022042445/https://www.novelguide.com/julius-caesar/index.html\nPygmalion\thttps://web.archive.org/web/20210211232247/https://www.novelguide.com/pygmalion/introduction\nA Doll's House\thttps://web.archive.org/web/20210213005530/https://www.novelguide.com/a-dolls-house\nHedda Gabler\thttps://web.archive.org/web/20201204111334/https://www.novelguide.com/hedda-gabler/index.html\nDaisy Miller\thttps://web.archive.org/web/20210212220747/https://www.novelguide.com/daisy-miller/index.html\nAn Enemy of the People\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/an-enemy-of-the-people\nAlice in Wonderland\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/alice-in-wonderland/index\nWhite Fang\thttps://web.archive.org/web/20210228134637/https://www.novelguide.com/white-fang/index.html\nCymbeline\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/cymbeline\nAll's Well That Ends Well\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/alls-well-that-ends-well\nCyrano de Bergerac\thttps://web.archive.org/web/20210212220448/https://www.novelguide.com/cyrano-de-bergerac/index\nKing Lear\thttps://web.archive.org/web/20210228115510/https://www.novelguide.com/king-lear/index.html\nUnder the Greenwood Tree\thttps://web.archive.org/web/20210214151736/https://www.novelguide.com/under-the-greenwood-tree\nOf Human Bondage\thttps://web.archive.org/web/20201204103600/https://www.novelguide.com/of-human-bondage\nBeowulf\thttps://web.archive.org/web/20201020010735/https://www.novelguide.com/beowulf\nAnthem\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/anthem\nEthan Frome\thttps://web.archive.org/web/20210215030716/https://www.novelguide.com/ethan-frome/index.html\nInvisible Man\thttps://web.archive.org/web/20201205184924/https://www.novelguide.com/invisible-man/index.html\nMuch Ado About Nothing\thttps://web.archive.org/web/20210212220253/https://www.novelguide.com/much-ado-about-nothing/index.html\nAntony And Cleopatra\thttps://web.archive.org/web/20200109120617/http://www.novelguide.com/antony-and-cleopatra/index\n"
},
{
"path": "scripts/data_collection/pinkmonkey/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom unidecode import unidecode\nimport time\n\nfrom urllib.error import HTTPError, URLError\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/pinkmonkey/summaries'\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\n#Always create a new errors file when starting to run the script\nf_errors = open(\"section_errors.txt\",\"w\")\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\ndef hasNumbers(inputString):\n return any(char.isdigit() for char in inputString)\n\ndef chapter_section_check(link_text_lower, link_text_not_lower):\n return 'chapter' in link_text_lower or 'scene' in link_text_lower\\\n or 'Act' in link_text_not_lower or 'part' in link_text_lower or 'prologue' in link_text_lower or 'epilogue' in link_text_lower\\\n or 'story' in link_text_lower or 'preface' in link_text_lower or 'Section' in link_text_not_lower\n\n\ndef remove_toc(text):\n pat = '((.*)(table[ ]{1,}of contents.*))'\n\n if re.match(pat, text, re.IGNORECASE):\n to_replace = re.match(pat, text, re.IGNORECASE).group(3)\n\n text = text.replace(to_replace, \"\")\n \n return text\n\ndef get_overview_paragraphs(overview, specific_summary_dir):\n overview_paragraphs = []\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n except Exception as e:\n print (e)\n time.sleep(4)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n except Exception as e:\n print (\"Overview not found: \", e, overview)\n\n # with open(\"section_errors.txt\",\"a\") as f:\n f_errors.write(overview + \"\\t\" + \"Overview\" + \"\\t\" + specific_summary_dir + \"\\n\")\n\n return overview_paragraphs\n\n flag = 0\n pat = \"(.*\\(synopsis\\))\"\n\n paragraphs = soup.findAll([\"p\",\"h3\"])\n\n iframe_text = \"Your browser does not support the IFRAME tag.\"\n\n for ix, paragraph in enumerate(paragraphs):\n overview_text = paragraph.text.strip().replace(iframe_text, \"\").replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n if re.match(pat, overview_text, re.IGNORECASE):\n break\n\n if re.match(pat, overview_text, re.IGNORECASE):\n to_replace = re.match(pat, overview_text, re.IGNORECASE).group(1)\n\n overview_text = overview_text.replace(to_replace, \"\")\n\n overview_text = remove_toc(overview_text)\n overview_text = unidecode(overview_text)\n\n overview_text = \". \".join([line.strip().rstrip() for line in overview_text.split('. ')])\n\n return overview_text\n\ndef save_section_para(section_text, section_title, section_link, specific_summary_dir, index):\n\n section_text = remove_toc(section_text)\n section_text = remove_toc(section_text)\n\n section_dict = {}\n section_dict[\"name\"] = section_title\n section_dict[\"summary\"] = section_text\n section_dict[\"analysis\"] = \"\"\n section_dict[\"url\"] = section_link\n\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % index)\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)\n\ndef get_section_paragraphs(page_url, specific_summary_dir):\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n section_paragraphs = []\n all_links = []\n section_links = []\n flag = 0\n\n one_level_up_url = os.path.dirname(page_url)\n\n all_links = soup.findAll(\"a\")\n\n overview_exists = 0\n for link in all_links:\n\n link_text_not_lower = link.text.strip().replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n link_text_lower = link.text.strip().lower().replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n if \"summaries\" in link_text_lower or 'synopsis' in link_text_lower or 'plot' in link_text_lower or chapter_section_check(link_text_lower, link_text_not_lower):\n \n section_path = os.path.join(one_level_up_url, link.get(\"href\"))\n section_links.append((link.text.strip().rstrip(), section_path))\n\n if 'synopsis' in link_text_lower or 'plot' in link_text_lower:\n overview_exists = 1\n\n overview_found = 0\n index = -1\n\n for link_text, link in section_links:\n link_text = link_text.replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n link_text_lower = link_text.strip().rstrip().lower().replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n link_text_not_lower = link_text.strip().rstrip().replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n\n\n #Fetch overview first\n if overview_exists and ('synopsis' in link_text_lower or 'plot' in link_text_lower) and overview_found == 0:\n\n overview = link\n overview_title = link_text\n\n print (overview_title, overview)\n \n overview_text = get_overview_paragraphs(overview, specific_summary_dir)\n\n overview_dict = {}\n overview_dict[\"name\"] = \"overview\"\n overview_dict[\"summary\"] = overview_text\n overview_dict[\"analysis\"] = \"\"\n overview_dict[\"url\"] = overview\n\n output_fname = os.path.join(specific_summary_dir, \"overview.json\")\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(overview_dict, fp)\n\n overview_found = 1\n continue\n\n if (overview_found == 1 or not overview_exists) and chapter_section_check(link_text_lower, link_text_not_lower):\n\n chapter_url = link\n \n print(link_text, chapter_url)\n\n index += 1\n \n try:\n chapter_soup = BeautifulSoup(urllib.request.urlopen(chapter_url), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n chapter_soup = BeautifulSoup(urllib.request.urlopen(chapter_url), \"html.parser\")\n except Exception as e:\n print (chapter_url, e)\n f_errors.write(chapter_url + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n continue\n except Exception as e:\n print (e)\n time.sleep(4)\n try:\n chapter_soup = BeautifulSoup(urllib.request.urlopen(chapter_url), \"html.parser\")\n except Exception as e:\n print (\"Chapter not found: \", e, chapter_url)\n\n # with open(\"section_errors.txt\",\"a\") as f:\n f_errors.write(str(index) + \"\\t\" + chapter_url + \"\\t\" + link_text + \"\\t\" + specific_summary_dir + \"\\n\")\n\n continue\n\n chapter_paras = chapter_soup.findAll([\"p\", \"h3\"])\n\n iframe_text = \"Your browser does not support the IFRAME tag.\"\n\n section_text_paras = []\n\n for ix, chapter_para in enumerate(chapter_paras):\n try:\n section_text = chapter_para.text.strip().replace(iframe_text, \"\").replace(\"\\r\\n\",\" \").replace(\"\\n\",\" \")\n\n section_text_paras.append(unidecode(section_text))\n \n except Exception as e: # No text inside the para HTML\n print (\"Summary not found: \", e, chapter_url)\n\n f_errors.write(str(index) + \"\\t\" + chapter_url + \"\\t\" + link_text + \"\\t\" + specific_summary_dir + \"\\n\")\n continue\n \n section_text = ' '.join(section_text_paras)\n\n section_text = \". \".join([line.strip().rstrip() for line in section_text.split('. ')])\n section_text = \" \".join([word.strip() for word in section_text.split()])\n\n # Remove obvious noise from the summary text\n pat_toc = '.*?(Table of Contents(.*$))'\n\n if re.match(pat_toc, section_text):\n to_replace = re.match(pat_toc, section_text).group(1)\n section_text = section_text.replace(to_replace, \"\")\n\n section_text = section_text.replace(\"Help / FAQ\", \"\").strip() # why no remove?\n section_text = section_text.replace(\"Please Take our User Survey\", \"\").strip() # why no remove?\n \n section_title = link_text\n\n # print (\"section_text SAVED: \", section_text)\n\n save_section_para(section_text, section_title, chapter_url, specific_summary_dir, index)\n \n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n print('\\n>>> {}. {} - {} <<<'.format(k, title, page_url))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory, skipping.\")\n # continue\n\n # Parse page\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n f_errors.write(page_url + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n continue\n except Exception as e:\n print (\"page not found: \", e)\n continue\n \n get_section_paragraphs(page_url, specific_summary_dir)\n\n \n\n"
},
{
"path": "scripts/data_collection/pinkmonkey/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io, string\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\n\n# Book summaries from barronsbooknotes redirect to pinkmonkey\nMAIN_SITE = 'https://web.archive.org/web/20180820042551/http://barronsbooknotes.com/'\nSEED_URL = 'https://web.archive.org/web/20180820042551/http://barronsbooknotes.com/'\n\nalphabet_list = string.ascii_lowercase\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n\n scraped_links = []\n collected = []\n\n for char in alphabet_list:\n books_page = seed_page + char + \".html\"\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n except Exception as e:\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n except Exception as e:\n print (\"Skipping: \", books_page, e)\n errors_file.write(books_page + \"\\t\" + str(e) + \"\\n\")\n continue\n\n items = soup.findAll(\"div\", {\"align\": \"left\"})\n books = items[0].findAll(\"a\")\n\n # # # Go over each section\n for index, item in enumerate(books):\n # Parse section to get bullet point text\n try:\n\n item_title = \" \".join(item.text.strip().split())\n item_url = [char for char in item.get(\"href\")]\n item_url[-5] = '2'\n\n item_url = \"\".join(item_url)\n\n if item_title != \"\":\n\n print (\"item_title: \", item_title)\n print (\"item_url: \", item_url.strip(), \"\\n\")\n\n scraped_links.append({\n \"title\": \" \".join(item_title.split()),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n\n except Exception as e:\n print (\"Link not found\")\n print (e)\n \n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/pinkmonkey/literature_links.tsv.pruned",
"content": "Portrait of a Lady\thttps://web.archive.org/web/20180820033733/http://www.pinkmonkey.com/booknotes/monkeynotes/pmPortraitLady02.asp\nCymbeline\thttps://web.archive.org/web/20180820053313/http://www.pinkmonkey.com/booknotes/monkeynotes/pmCymbeline02.asp\nMan and Superman\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmManSuperman02.asp\nLove's Labour's Lost\thttps://web.archive.org/web/20180820051943/http://www.pinkmonkey.com/booknotes/monkeynotes/pmLovesLabours02.asp\nCoriolanus\thttps://web.archive.org/web/20180820053313/http://www.pinkmonkey.com/booknotes/monkeynotes/pmCoriolanus02.asp\nRichard II\thttps://web.archive.org/web/20180820044717/http://www.pinkmonkey.com/booknotes/monkeynotes/pmRichard02.asp\nWuthering Heights\thttps://web.archive.org/web/20210416215230/http://www.pinkmonkey.com/booknotes/monkeynotes/pmWuthering02.asp\nFrankenstein\thttps://web.archive.org/web/20180820054109/http://www.pinkmonkey.com/booknotes/monkeynotes/pmFrankenstein02.asp\nWinesburg, Ohio\thttps://web.archive.org/web/20180820045946/http://www.pinkmonkey.com/booknotes/monkeynotes/pmWinesburg02.asp\nMadame Bovary\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMadameBovary02.asp\nOthello\thttps://web.archive.org/web/20210416215230/http://www.pinkmonkey.com/booknotes/monkeynotes/pmOthello02.asp\nSense and Sensibility\thttps://web.archive.org/web/20180820034609/http://www.pinkmonkey.com/booknotes/monkeynotes/pmSenseSensibility02.asp\nTess of the D'Urbervilles\thttps://web.archive.org/web/20180820050202/http://www.pinkmonkey.com/booknotes/monkeynotes/pmTessD02.asp\nLord Jim\thttps://web.archive.org/web/20180820051943/http://www.pinkmonkey.com/booknotes/monkeynotes/pmLordJim02.asp\nDracula\thttps://web.archive.org/web/20180820041120/http://www.pinkmonkey.com/booknotes/monkeynotes/pmDracula02.asp\nJane Eyre\thttps://web.archive.org/web/20180820051427/http://www.pinkmonkey.com/booknotes/monkeynotes/pmJaneEyre02.asp\nMain Street\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMainStreet02.asp\nEmma\thttps://web.archive.org/web/20180820053844/http://www.pinkmonkey.com/booknotes/monkeynotes/pmEmma02.asp\nAlice's Adventures In Wonderland\thttps://web.archive.org/web/20180820042551/http://www.pinkmonkey.com/booknotes/monkeynotes/pmAlice02.asp\nJude the Obscure\thttps://web.archive.org/web/20180820051427/http://www.pinkmonkey.com/booknotes/monkeynotes/pmJude02.asp\nMiddlemarch\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMiddle02.asp\nSiddhartha\thttps://web.archive.org/web/20180820034609/http://www.pinkmonkey.com/booknotes/monkeynotes/pmSiddhartha02.asp\nKidnapped\thttps://web.archive.org/web/20180820035817/http://www.pinkmonkey.com/booknotes/monkeynotes/pmKidnapped02.asp\nDr. Jekyll and Mr. Hyde\thttps://web.archive.org/web/20180820041120/http://www.pinkmonkey.com/booknotes/monkeynotes/pmJekyll02.asp\nMy Antonia\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMyAntonia02.asp\nHamlet\thttps://web.archive.org/web/20180820040441/http://www.pinkmonkey.com/booknotes/monkeynotes/pmHamlet02.asp\nA Midsummer Night's Dream\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMidsummer02.asp\nAround the World in Eighty Days\thttps://web.archive.org/web/20180820042551/http://www.pinkmonkey.com/booknotes/monkeynotes/pmAroundWorld02.asp\nHeart of Darkness\thttps://web.archive.org/web/20180820040441/http://www.pinkmonkey.com/booknotes/monkeynotes/pmHeartDarkness02.asp\nTypee\thttps://web.archive.org/web/20180820050202/http://www.pinkmonkey.com/booknotes/monkeynotes/pmTypee02.asp\nMaggie: A Girl of the Streets\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMaggie02.asp\nHenry VIII\thttps://web.archive.org/web/20180820040441/http://www.pinkmonkey.com/booknotes/monkeynotes/pmHenry802.asp\nMacbeth\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMacbeth02.asp\nTroilus and Cressida\thttps://web.archive.org/web/20180820050202/http://www.pinkmonkey.com/booknotes/monkeynotes/pmTroilus02.asp\nHenry IV, Part 1\thttps://web.archive.org/web/20180820040441/http://www.pinkmonkey.com/booknotes/monkeynotes/pmHenry4102.asp\nTitus Andronicus\thttps://web.archive.org/web/20180820050202/http://www.pinkmonkey.com/booknotes/monkeynotes/pmTitus02.asp\nJulius Caesar\thttps://web.archive.org/web/20180820051427/http://www.pinkmonkey.com/booknotes/monkeynotes/pmJuliusCaesar02.asp\nPygmalion\thttps://web.archive.org/web/20180820033733/http://www.pinkmonkey.com/booknotes/monkeynotes/pmPygmalion02.asp\nArms and the Man\thttps://web.archive.org/web/20180820042551/http://www.pinkmonkey.com/booknotes/monkeynotes/pmArmsMan02.asp\nHedda Gabler\thttps://web.archive.org/web/20180820040441/http://www.pinkmonkey.com/booknotes/monkeynotes/pmHeddaGabler02.asp\nAn Enemy of the People\thttps://web.archive.org/web/20180820053844/http://www.pinkmonkey.com/booknotes/monkeynotes/pmEnemyPeople02.asp\nAll's Well That Ends Well\thttps://web.archive.org/web/20180820042551/http://www.pinkmonkey.com/booknotes/monkeynotes/pmAllsWell02.asp\nMajor Barbara\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMajorBarbara02.asp\nCyrano De Bergerac\thttps://web.archive.org/web/20180820053313/http://www.pinkmonkey.com/booknotes/monkeynotes/pmCyrano02.asp\nKing Lear\thttps://web.archive.org/web/20180820035817/http://www.pinkmonkey.com/booknotes/monkeynotes/pmKingLear02.asp\nReturn of the Native\thttps://web.archive.org/web/20180820044717/http://www.pinkmonkey.com/booknotes/monkeynotes/pmReturnNative02.asp\nNotes From Underground\thttps://web.archive.org/web/20180820034941/http://www.pinkmonkey.com/booknotes/monkeynotes/pmNotesFrom02.asp\nUnder the Greenwood Tree\thttps://web.archive.org/web/20180820032827/http://www.pinkmonkey.com/booknotes/monkeynotes/pmGreenwood02.asp\nOf Human Bondage\thttps://web.archive.org/web/20180820052247/http://www.pinkmonkey.com/booknotes/monkeynotes/pmOfHuman02.asp\nEthan Frome\thttps://web.archive.org/web/20180820053844/http://www.pinkmonkey.com/booknotes/monkeynotes/pmEthanFrome02.asp\nTimon of Athens\thttps://web.archive.org/web/20180820050202/http://www.pinkmonkey.com/booknotes/monkeynotes/pmTimon02.asp\nMuch Ado About Nothing\thttps://web.archive.org/web/20180820035130/http://www.pinkmonkey.com/booknotes/monkeynotes/pmMuchado02.asp\nVanity Fair\thttps://web.archive.org/web/20180820033155/http://www.pinkmonkey.com/booknotes/monkeynotes/pmVanity02.asp\n"
},
{
"path": "scripts/data_collection/shmoop/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\nfrom urllib.error import HTTPError, URLError\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\nfrom multiprocessing import Pool\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/shmoop/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210225092515/https://www.shmoop.com/'\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\nerrors_file = open(\"section_errors.txt\",\"w\")\n\ndef wrap_data(name, summary, analysis, url):\n return {\n \"name\": name,\n \"summary\": summary,\n \"analysis\": analysis,\n \"url\": url\n }\n\n\ndef get_summary(summary_infos):\n print (\"summary_infos: \", summary_infos)\n # For each summary info\n # for k, (title, url) in enumerate(summary_infos):\n title = summary_infos[0]\n url = summary_infos[1]\n print('\\n>>> {} <<<'.format(title))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory, skipping.\")\n # continue\n\n # Parse page\n html_address = urllib.parse.urljoin(url + \"/\", \"summary\")\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(html_address), \"html.parser\")\n except Exception as e:\n time.sleep(5)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(html_address), \"html.parser\")\n except Exception as e:\n print (html_address, e)\n errors_file.write(html_address + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n return\n\n # Parse general summary\n overview_section = soup.find(\"div\", {\"data-class\": \"SHPlotOverviewSection\"})\n overview_section = soup.find(\"div\", {\"class\": \"content-wrapper\"})\n overview_summary_paragraphs = [paragraph.text.strip() for paragraph in overview_section.findAll(\"p\")]\n overview_summary = \"\".join(overview_summary_paragraphs)\n\n overview_data = wrap_data(\"Overview\", overview_summary, None, str(html_address))\n output_fname = os.path.join(specific_summary_dir, 'overview.txt')\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(overview_data))\n\n # Parse sections summary \n summary_sections = [(link.text, urllib.parse.urljoin(MAIN_SITE, link.get(\"href\"))) for link in soup.find(\"div\", {\"class\": \"nav-menu\"}).findAll(\"a\", href=True) if \"summary\" in link.get(\"href\")]\n # Go over each section\n for index, (section_title, section_url) in enumerate(summary_sections):\n output_fname = os.path.join(specific_summary_dir, \"section_%d.txt\" % index)\n\n print (section_title, section_url)\n \n # Parse section to get bullet point text\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section_url), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section_url), \"html.parser\")\n except Exception as e:\n print (section_url, e)\n errors_file.write(section_url + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n \n except Exception as e:\n print (section_url, e)\n errors_file.write(section_url + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n\n try:\n section_points = soup.find(\"div\", {\"data-element\": \"collapse_target\"})\n section_text = \"\".join([bullet.text.strip() for bullet in section_points.findAll(\"li\")])\n\n # Try alternate\n if section_text == '':\n section_text = \"\".join([bullet.text.strip() for bullet in section_points.findAll(\"p\")])\n\n section_data = wrap_data(section_title, section_text, None, str(section_url))\n # Save in a file\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(section_data))\n print (\"Saved to file\")\n except Exception as e:\n print (section_url, e)\n errors_file.write(section_url + \"\\t\" + str(e))\n errors_file.write(\"\\n\")\n continue\n \n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\nwith Pool(1) as p:\n p.map(get_summary, summary_infos)\n"
},
{
"path": "scripts/data_collection/shmoop/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210225092515/https://www.shmoop.com/study-guides'\n\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef generate_page_links(base_url, category_name, max_pages):\n return [os.path.join(base_url, category_name, \"index?p=%d\" % page_id) for page_id in range(1, max_pages+1)]\n\n\ndef scrape_index_pages(links):\n# For each summary info\n error_files, error_titles = [], []\n scraped_links = []\n\n for k, page_url in enumerate(links):\n print('>>> {}. {} <<<'.format(k, page_url))\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (\"Skipping: \", page_url)\n errors_file.write(page_url + \"\\t\" + str(e) + \"\\n\")\n continue\n\n items = soup.findAll(\"div\", {\"class\" : \"item\"})\n print(\"Found %d items.\" % len(items))\n\n # Go over each section\n for index, item in enumerate(items):\n # Parse section to get bullet point text\n item_title = item.find(\"div\", {\"class\": \"item-info\"}).text\n item_url = item.find(\"a\", {\"class\": \"details\"}).get(\"href\")\n\n scraped_links.append({\n \"title\": item_title.strip(),\n \"url\": item_url.strip()\n })\n return scraped_links\n\n# generate literature links\n\nworks_list = generate_page_links(MAIN_SITE, \"literature\", 95)\nscraped_data = scrape_index_pages(works_list)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/shmoop/literature_links.tsv.pruned",
"content": "Romeo and Juliet\thttps://web.archive.org/web/20210624200133/https://www.shmoop.com/study-guides/literature/romeo-and-juliet\nJulius Caesar\thttps://web.archive.org/web/20210624193711/https://www.shmoop.com/study-guides/literature/julius-caesar\nIvanhoe\thttps://web.archive.org/web/20210624193654/https://www.shmoop.com/study-guides/literature/ivanhoe\nThe Scarlet Letter\thttps://web.archive.org/web/20210624201606/https://www.shmoop.com/study-guides/literature/scarlet-letter\nThe Importance of Being Earnest\thttps://web.archive.org/web/20210624201341/https://www.shmoop.com/study-guides/literature/importance-of-being-earnest\nHowards End\thttps://web.archive.org/web/20210624193615/https://www.shmoop.com/study-guides/literature/howards-end\nWhite Fang\thttps://web.archive.org/web/20200929150717/https://www.shmoop.com/study-guides/literature/white-fang\nCoriolanus\thttps://web.archive.org/web/20200928191345/https://www.shmoop.com/study-guides/literature/coriolanus\nThe Merry Wives of Windsor\thttps://web.archive.org/web/20210624201500/https://www.shmoop.com/study-guides/literature/merry-wives-of-windsor\nA Doll's House\thttps://web.archive.org/web/20210304225252/https://www.shmoop.com/study-guides/literature/dolls-house\nSons and Lovers\thttps://web.archive.org/web/20210624200850/https://www.shmoop.com/study-guides/literature/sons-and-lovers\nOthello\thttps://web.archive.org/web/20210624194547/https://www.shmoop.com/study-guides/literature/othello\nTitus Andronicus\thttps://web.archive.org/web/20210624201826/https://www.shmoop.com/study-guides/literature/titus-andronicus-shakespeare\nLove's Labour's Lost\thttps://web.archive.org/web/20210624193742/https://www.shmoop.com/study-guides/literature/loves-labours-lost\nEmma\thttps://web.archive.org/web/20201127211107/https://www.shmoop.com/study-guides/literature/emma\nAn Enquiry Concerning the Principles of Morals\thttps://web.archive.org/web/20200928194100/https://www.shmoop.com/study-guides/literature/enquiry-concerning-the-principles-of-morals\nKidnapped\thttps://web.archive.org/web/20210624193722/https://www.shmoop.com/study-guides/literature/kidnapped\nThe Goose Girl\thttps://web.archive.org/web/20210624201228/https://www.shmoop.com/study-guides/literature/the-goose-girl\nThe Wonderful Wizard of Oz\thttps://web.archive.org/web/20210624201753/https://www.shmoop.com/study-guides/literature/wonderful-wizard-of-oz-book\nThe Spanish Tragedy\thttps://web.archive.org/web/20210624201644/https://www.shmoop.com/study-guides/literature/the-spanish-tragedy\nThe Mysteries of Udolpho\thttps://web.archive.org/web/20210624201500/https://www.shmoop.com/study-guides/literature/mysteries-of-udolpho\nAdam Bede\thttps://web.archive.org/web/20201125114409/https://www.shmoop.com/study-guides/literature/adam-bede\nTess of the D'Urbervilles\thttps://web.archive.org/web/20210624200938/https://www.shmoop.com/study-guides/literature/tess-of-the-durbervilles\nThe Prince\thttps://web.archive.org/web/20210624201604/https://www.shmoop.com/study-guides/literature/prince-machiavelli\nSense and Sensibility\thttps://web.archive.org/web/20210624200213/https://www.shmoop.com/study-guides/literature/sense-and-sensibility\nFar From the Madding Crowd\thttps://web.archive.org/web/20210624192652/https://www.shmoop.com/study-guides/literature/far-from-the-madding-crowd\nThe Brothers Karamazov\thttps://web.archive.org/web/20210624201022/https://www.shmoop.com/study-guides/literature/brothers-karamazov\nThe Picture of Dorian Gray\thttps://web.archive.org/web/20210624201544/https://www.shmoop.com/study-guides/literature/picture-dorian-gray\nLord Jim\thttps://web.archive.org/web/20210624193742/https://www.shmoop.com/study-guides/literature/lord-jim\nTwelfth Night, or What You Will\thttps://web.archive.org/web/20210624202234/https://www.shmoop.com/study-guides/literature/twelfth-night\nThe Red and the Black\thttps://web.archive.org/web/20210624201556/https://www.shmoop.com/study-guides/literature/red-and-the-black\nThe Comedy of Errors\thttps://web.archive.org/web/20210624201036/https://www.shmoop.com/study-guides/literature/comedy-of-errors\nThe White Devil\thttps://web.archive.org/web/20210624201726/https://www.shmoop.com/study-guides/literature/the-white-devil\nThe Tempest\thttps://web.archive.org/web/20210624201650/https://www.shmoop.com/study-guides/literature/tempest\nThe Tragedy of Antony and Cleopatra\thttps://web.archive.org/web/20210624201736/https://www.shmoop.com/study-guides/literature/antony-cleopatra\nUncle Vanya\thttps://web.archive.org/web/20210624202234/https://www.shmoop.com/study-guides/literature/uncle-vanya\nThe Portrait of a Lady\thttps://web.archive.org/web/20210624201604/https://www.shmoop.com/study-guides/literature/portrait-of-a-lady\nDracula\thttps://web.archive.org/web/20201127213435/https://www.shmoop.com/study-guides/literature/dracula\nJane Eyre\thttps://web.archive.org/web/20210624193654/https://www.shmoop.com/study-guides/literature/jane-eyre\nThe Hound of the Baskervilles\thttps://web.archive.org/web/20210624201316/https://www.shmoop.com/study-guides/literature/hound-of-the-baskervilles\nMadame Bovary\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/madame-bovary\nThe Aeneid\thttps://web.archive.org/web/20210624201011/https://www.shmoop.com/study-guides/literature/aeneid\nMain Street\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/main-street\nOliver Twist\thttps://web.archive.org/web/20210624193914/https://www.shmoop.com/study-guides/literature/oliver-twist\nThe House of the Seven Gables\thttps://web.archive.org/web/20210624201316/https://www.shmoop.com/study-guides/literature/house-seven-gables\nVanity Fair\thttps://web.archive.org/web/20200929144928/https://www.shmoop.com/study-guides/literature/vanity-fair-thackeray\nLittle Women\thttps://web.archive.org/web/20210624193802/https://www.shmoop.com/study-guides/literature/little-women\nBabbitt\thttps://web.archive.org/web/20210116190210/https://www.shmoop.com/study-guides/literature/babbitt\nA Tale of Two Cities\thttps://web.archive.org/web/20201205220634/https://www.shmoop.com/study-guides/literature/tale-of-two-cities\nThe House of Mirth\thttps://web.archive.org/web/20210624201316/https://www.shmoop.com/study-guides/literature/house-of-mirth\nPersuasion\thttps://web.archive.org/web/20210624194445/https://www.shmoop.com/study-guides/literature/persuasion\nGulliver's Travels\thttps://web.archive.org/web/20210624193541/https://www.shmoop.com/study-guides/literature/gullivers-travels\nWuthering Heights\thttps://web.archive.org/web/20200919013354/https://www.shmoop.com/study-guides/literature/wuthering-heights\nFrankenstein\thttps://web.archive.org/web/20210624192730/https://www.shmoop.com/study-guides/literature/frankenstein\nMansfield Park\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/mansfield-park\nMiddlemarch\thttps://web.archive.org/web/20210624193915/https://www.shmoop.com/study-guides/literature/middlemarch\nSiddhartha\thttps://web.archive.org/web/20210624200823/https://www.shmoop.com/study-guides/literature/siddhartha\nTreasure Island\thttps://web.archive.org/web/20210624201815/https://www.shmoop.com/study-guides/literature/treasure-island-book\nDavid Copperfield\thttps://web.archive.org/web/20201021164515/https://www.shmoop.com/study-guides/literature/david-copperfield\nSister Carrie\thttps://web.archive.org/web/20210624200857/https://www.shmoop.com/study-guides/literature/sister-carrie\nThe Turn of the Screw\thttps://web.archive.org/web/20210624201736/https://www.shmoop.com/study-guides/literature/turn-of-the-screw\nCandide\thttps://web.archive.org/web/20201026164007/https://www.shmoop.com/study-guides/literature/candide\nParadise Lost\thttps://web.archive.org/web/20210624193940/https://www.shmoop.com/study-guides/literature/paradise-lost\nNorthanger Abbey\thttps://web.archive.org/web/20210624193925/https://www.shmoop.com/study-guides/literature/northanger-abbey\nMy Antonia\thttps://web.archive.org/web/20210624193939/https://www.shmoop.com/study-guides/literature/my-antonia\nA Vindication of the Rights of Woman\thttps://web.archive.org/web/20201205220634/https://www.shmoop.com/study-guides/literature/a-vindication-of-the-rights-of-woman\nThe Life of Timon of Athens\thttps://web.archive.org/web/20210624201506/https://www.shmoop.com/study-guides/literature/timon-of-athens\nHamlet\thttps://web.archive.org/web/20210624193541/https://www.shmoop.com/study-guides/literature/hamlet\nA Midsummer Night's Dream\thttps://web.archive.org/web/20210304230026/https://www.shmoop.com/study-guides/literature/midsummer-nights-dream\nA Room with a View\thttps://web.archive.org/web/20210304213804/https://www.shmoop.com/study-guides/literature/room-with-a-view\nAround the World in Eighty Days\thttps://web.archive.org/web/20210116190210/https://www.shmoop.com/study-guides/literature/around-the-world-in-eighty-days\nWhat Maisie Knew\thttps://web.archive.org/web/20210624202247/https://www.shmoop.com/study-guides/literature/what-maisie-knew\nWhere Angels Fear to Tread\thttps://web.archive.org/web/20200929150717/https://www.shmoop.com/study-guides/literature/where-angels-fear-to-tread\nHeart of Darkness\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/heart-of-darkness\nLeviathan\thttps://web.archive.org/web/20210624193802/https://www.shmoop.com/study-guides/literature/leviathan\nFrom the Earth to the Moon\thttps://web.archive.org/web/20210624192803/https://www.shmoop.com/study-guides/literature/from-the-earth-to-the-moon\nThe Power and the Glory\thttps://web.archive.org/web/20210624201604/https://www.shmoop.com/study-guides/literature/the-power-and-the-glory\nNotes from the Underground\thttps://web.archive.org/web/20210624193925/https://www.shmoop.com/study-guides/literature/notes-from-underground\nThe Return of the Native\thttps://web.archive.org/web/20210624201543/https://www.shmoop.com/study-guides/literature/return-of-native\nThe Secret Agent\thttps://web.archive.org/web/20210624201606/https://www.shmoop.com/study-guides/literature/the-secret-agent\nThe Confidence-Man\thttps://web.archive.org/web/20210624201036/https://www.shmoop.com/study-guides/literature/confidence-man\nTypee\thttps://web.archive.org/web/20210624202234/https://www.shmoop.com/study-guides/literature/typee\nAnne of Green Gables\thttps://web.archive.org/web/20200928194643/https://www.shmoop.com/study-guides/literature/anne-of-green-gables\nA Little Princess\thttps://web.archive.org/web/20210304230026/https://www.shmoop.com/study-guides/literature/little-princess\nMaggie: A Girl of the Streets\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/maggie-a-girl-of-the-streets\nThe Monkey's Paw\thttps://web.archive.org/web/20210624201500/https://www.shmoop.com/study-guides/literature/monkeys-paw\nThe Red Badge of Courage\thttps://web.archive.org/web/20210624201556/https://www.shmoop.com/study-guides/literature/red-badge-of-courage\nO Pioneers!\thttps://web.archive.org/web/20210624193914/https://www.shmoop.com/study-guides/literature/o-pioneers\nThe Mill on the Floss\thttps://web.archive.org/web/20210624201500/https://www.shmoop.com/study-guides/literature/mill-on-the-floss\nHenry VIII\thttps://web.archive.org/web/20210624193614/https://www.shmoop.com/study-guides/literature/henry-viii\nThe Time Machine\thttps://web.archive.org/web/20210624201736/https://www.shmoop.com/study-guides/literature/time-machine-hg-wells\nKim\thttps://web.archive.org/web/20210624193722/https://www.shmoop.com/study-guides/literature/kim-rudyard-kipling\nThe Invisible Man\thttps://web.archive.org/web/20210624201403/https://www.shmoop.com/study-guides/literature/invisible-man-wells\nLittle Dorrit\thttps://web.archive.org/web/20210624193802/https://www.shmoop.com/study-guides/literature/little-dorrit\nIncidents in the Life of a Slave Girl\thttps://web.archive.org/web/20210624193640/https://www.shmoop.com/study-guides/literature/incidents-life-slave-girl\nHenry IV Part 2\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/henry-iv-part-2\nMacbeth\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/macbeth\nTroilus and Cressida\thttps://web.archive.org/web/20210624201815/https://www.shmoop.com/study-guides/literature/troilus-cressida\nRichard II\thttps://web.archive.org/web/20210624200133/https://www.shmoop.com/study-guides/literature/richard-ii\nHenry IV Part 1\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/henry-iv-part-1\nKing John\thttps://web.archive.org/web/20210624193722/https://www.shmoop.com/study-guides/literature/king-john\nHenry VI Part 1\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/henry-vi-part-1\nRichard III\thttps://web.archive.org/web/20210624200133/https://www.shmoop.com/study-guides/literature/richard-iii\nHenry V\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/henry-v\nThe Two Gentlemen of Verona\thttps://web.archive.org/web/20210624201802/https://www.shmoop.com/study-guides/literature/gentlemen-of-verona\nPygmalion\thttps://web.archive.org/web/20210624200145/https://www.shmoop.com/study-guides/literature/pygmalion\nThe Boxcar Children\thttps://web.archive.org/web/20210624201022/https://www.shmoop.com/study-guides/literature/boxcar-children\nThe Piazza Tales\thttps://web.archive.org/web/20210624201544/https://www.shmoop.com/study-guides/literature/the-piazza-tales\nThe Return of Sherlock Holmes\thttps://web.archive.org/web/20210624201543/https://www.shmoop.com/study-guides/literature/return-of-sherlock-holmes\nGhosts\thttps://web.archive.org/web/20210624192803/https://www.shmoop.com/study-guides/literature/ghosts-ibsen\nHedda Gabler\thttps://web.archive.org/web/20210624193540/https://www.shmoop.com/study-guides/literature/hedda-gabler\nDaisy Miller\thttps://web.archive.org/web/20210624192036/https://www.shmoop.com/study-guides/literature/daisy-miller\nTartuffe\thttps://web.archive.org/web/20210624200938/https://www.shmoop.com/study-guides/literature/tartuffe\nAn Enemy of the People\thttps://web.archive.org/web/20200928194100/https://www.shmoop.com/study-guides/literature/enemy-of-the-people\nMeasure for Measure\thttps://web.archive.org/web/20210624193815/https://www.shmoop.com/study-guides/literature/measure-for-measure\nThe Seagull\thttps://web.archive.org/web/20210624201606/https://www.shmoop.com/study-guides/literature/the-seagull\nIdylls of the King\thttps://web.archive.org/web/20210624193611/https://www.shmoop.com/study-guides/literature/idylls-of-the-king\nThe Merchant of Venice\thttps://web.archive.org/web/20210624201500/https://www.shmoop.com/study-guides/literature/merchant-of-venice\nMeditations\thttps://web.archive.org/web/20210624193815/https://www.shmoop.com/study-guides/literature/meditations\nThe Man in the Iron Mask\thttps://web.archive.org/web/20210624201428/https://www.shmoop.com/study-guides/literature/man-in-the-iron-mask\nAll's Well That Ends Well\thttps://web.archive.org/web/20201125115619/https://www.shmoop.com/study-guides/literature/alls-well-that-ends-well\nMajor Barbara\thttps://web.archive.org/web/20210624193757/https://www.shmoop.com/study-guides/literature/major-barbara\nThe Communist Manifesto\thttps://web.archive.org/web/20210624201036/https://www.shmoop.com/study-guides/literature/communist-manifesto\nAn Ideal Husband\thttps://web.archive.org/web/20200928194100/https://www.shmoop.com/study-guides/literature/an-ideal-husband\nThe Three Musketeers\thttps://web.archive.org/web/20210624201736/https://www.shmoop.com/study-guides/literature/three-musketeers\nBlack Beauty\thttps://web.archive.org/web/20201125120012/https://www.shmoop.com/study-guides/literature/black-beauty\nKing Lear\thttps://web.archive.org/web/20210624193722/https://www.shmoop.com/study-guides/literature/king-lear\nBleak House\thttps://web.archive.org/web/20201026155119/https://www.shmoop.com/study-guides/literature/bleak-house\nAnthem\thttps://web.archive.org/web/20200928194643/https://www.shmoop.com/study-guides/literature/anthem-ayn-rand\nEthan Frome\thttps://web.archive.org/web/20210624193104/https://www.shmoop.com/study-guides/literature/ethan-frome\nThe Jungle\thttps://web.archive.org/web/20210624201403/https://www.shmoop.com/study-guides/literature/the-jungle\nThe Ambassadors\thttps://web.archive.org/web/20210624201020/https://www.shmoop.com/study-guides/literature/the-ambassadors\nThe Age of Innocence\thttps://web.archive.org/web/20210624201011/https://www.shmoop.com/study-guides/literature/the-age-of-innocence\nMuch Ado About Nothing\thttps://web.archive.org/web/20210624193920/https://www.shmoop.com/study-guides/literature/much-ado-about-nothing\nThe Secret Garden\thttps://web.archive.org/web/20210624201606/https://www.shmoop.com/study-guides/literature/the-secret-garden\nThe Duchess of Malfi\thttps://web.archive.org/web/20210624201210/https://www.shmoop.com/study-guides/literature/the-duchess-of-malfi\nThe Taming of the Shrew\thttps://web.archive.org/web/20210624201650/https://www.shmoop.com/study-guides/literature/taming-of-the-shrew\nAs You Like It\thttps://web.archive.org/web/20210116190210/https://www.shmoop.com/study-guides/literature/as-you-like-it\n"
},
{
"path": "scripts/data_collection/sparknotes/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\nfrom urllib.error import HTTPError, URLError\nfrom multiprocessing import Pool\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/sparknotes/summaries'\n# Summary list info\nsummary_list_file = \"literature_links.tsv.pruned'\n\nf_errors = open(\"section_errors.txt\",\"w\")\n\ndef wrap_data(name, summary, analysis, url):\n return {\n \"name\": name,\n \"summary\": summary,\n \"analysis\": analysis,\n \"url\": url\n }\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\n# For each summary info\n\ndef get_summary(summary_info):\n\n title = summary_info[0]\n page_url = summary_info[1]\n print('\\n>>> {} <<<'.format(title))\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory, skipping.\")\n\n # Parse page\n overview_url = urllib.parse.urljoin(page_url, \"summary\")\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview_url), \"html.parser\")\n except Exception as e:\n time.sleep(5)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview_url), \"html.parser\")\n except Exception as e:\n print (\"Overview error: \" , overview_url, e)\n f_errors.write(title + \"\\t\" + overview_url + \"\\t\" + str(e) + \"\\n\")\n\n # Parse general summary\n overview_data = soup.find(\"div\", {\"id\": \"plotoverview\"})\n\n if overview_data:\n overview_summary_paragraphs = [paragraph.text.strip().replace(\"\\n\", \" \") for paragraph in overview_data.findAll(\"p\")]\n overview_summary = \"\\n\".join(overview_summary_paragraphs)\n\n overview_data = wrap_data(\"Overview\", overview_summary, None, str(overview_url))\n output_fname = os.path.join(specific_summary_dir, 'overview.txt')\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(overview_data))\n else:\n print (\"No Overview text found\")\n\n # Parse sections summary \n print (\"page_url: \", page_url)\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n summary_section = soup.find(\"span\", {\"id\": \"Summary\"}).find_parent(\"div\")\n except Exception as e:\n time.sleep(5)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n summary_section = soup.find(\"span\", {\"id\": \"Summary\"}).find_parent(\"div\")\n except Exception as e:\n print (page_url, e)\n f_errors.write(title + \"\\t\" + page_url + \"\\t\" + str(e) + \"\\n\")\n return\n\n summary_links = summary_section.findAll(\"a\")\n summary_links = [link.get(\"href\") for link in summary_links if \"section\" in link.get(\"href\")]\n\n for index, section in enumerate(summary_links):\n print('Parsing section: {}'.format(index))\n output_fname = os.path.join(specific_summary_dir, \"section_%d.txt\" % index)\n\n section_url = urllib.parse.urljoin(page_url, section)\n print (\"section_url: \", section_url)\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section_url), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section_url), \"html.parser\")\n except Exception as e:\n print (section_url, e)\n f_errors.write(section_url + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n continue\n except Exception as e:\n print (\"Here:\", section_url, e)\n f_errors.write(str(index) + \"\\t\" + section_url + \"\\t\" + str(e) + \"\\n\")\n continue\n\n subsection_links = soup.find(\"div\", {\"class\": \"interior-sticky-nav\"})\n\n if subsection_links == None:\n subsection_links = soup.find(\"div\", {\"class\": \"pagination-links\"})\n\n # print (\"subsection_links before: \", subsection_links)\n num_subsections = max(1, len(subsection_links.findAll(\"a\") if subsection_links else []))\n subsection_links = [\"page/%d/\" % page_ix for page_ix in range(1, num_subsections+1)]\n\n section_header = \" \".join([x.strip() for x in soup.title.string.replace(\" | SparkNotes\", \"\").split(\":\")[1:]])\n\n section_paragraphs = []\n\n for subsection_link in subsection_links:\n\n if subsection_link == 'page/1/':\n subsection_link = \"\" \n\n subsection_url = urllib.parse.urljoin(section_url, subsection_link)\n print (\"section_header and subsection_url: \", section_header, subsection_url)\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(subsection_url), \"html.parser\")\n except URLError as err:\n print (err, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(subsection_url), \"html.parser\")\n except Exception as e:\n print (subsection_url, e)\n f_errors.write(subsection_url + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n continue\n except Exception as e:\n print (\"subsection error: \", subsection_url, e)\n f_errors.write(str(index) + \"\\t\" + section_header + \"\\t\" + section_url + \"\\t\" + subsection_url + \"\\t\" + str(e) + \"\\n\")\n continue\n\n subsection_data = soup.find(\"div\", {\"id\": \"section\"})\n\n if subsection_data == None:\n\n # Try alternate way\n subsection_data = soup.find(\"div\", {\"class\": \"mainTextContent\"})\n \n if subsection_data == None:\n print (\"subsection_data is None\")\n f_errors.write(str(index) + \"\\t\" + section_header + \"\\t\" + section_url + \"\\t\" + subsection_url + \"\\t\" + \"No Data\" + \"\\n\")\n continue\n\n section_paragraphs.append(subsection_data.text.strip().replace(\"\\n\", \" \"))\n\n if section_paragraphs == []:\n continue\n \n section_text = \"\".join(section_paragraphs)\n\n if \"Summary:\" in section_text and \"Analysis:\" in section_text:\n section_text_split = section_text.split(\"Analysis:\")\n summary_text = \" \".join([summary for summary in section_text_split if \"Summary:\" in summary]).replace(\"Summary:\", \"\").strip()\n analysis_text = \" \".join([analysis for analysis in section_text_split if \"Summary:\" not in analysis]).replace(\"Analysis:\", \"\").strip()\n else:\n summary_text = section_text.replace(\"Summary:\", \"\").strip()\n analysis_text = None\n\n\n section_data = wrap_data(section_header, summary_text, analysis_text, section_url)\n print (\"Saving to file: \", output_fname)\n with open(output_fname, 'w', encoding=\"utf-8\") as f:\n f.write(json.dumps(section_data))\n\n\nwith Pool(1) as p:\n p.map(get_summary, summary_infos)\n"
},
{
"path": "scripts/data_collection/sparknotes/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210223175142/https://www.sparknotes.com/'\nSEED_URL = 'https://web.archive.org/web/20210223175142/https://www.sparknotes.com/lit'\n\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n scraped_links = []\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(seed_page), \"html.parser\")\n except Exception as e:\n print (\"Skipping: \", seed_page)\n errors_file.write(seed_page + \"\\t\" + str(e) + \"\\n\")\n return []\n\n items = soup.findAll(\"li\", {\"class\": \"hub-AZ-list__card hub-AZ-list__card--byTitle\"})\n print(\"Found %d items.\" % len(items))\n\n # Go over each section\n for index, item in enumerate(items):\n # Parse section to get bullet point text\n item_title = item.find(\"a\", href=True).text\n item_url = item.find(\"a\").get(\"href\")\n\n scraped_links.append({\n \"title\": item_title.strip().replace(\",\",\"\"),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/sparknotes/literature_links.tsv.pruned",
"content": "The House of the Seven Gables\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/sevengables/\nThe Scarlet Letter\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/scarlet/\nA Tale of Two Cities\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/a-tale-of-two-cities/\nThe Brothers Karamazov\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/brothersk/\nFar from the Madding Crowd\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/maddingcrowd/\nFrankenstein\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/frankenstein/\nThe Secret Garden\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/secretgarden/\nThe Taming of the Shrew\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/shrew/\nAntony and Cleopatra\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/antony/\nThe Prince\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/philosophy/prince/\nAs You Like It\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/asyoulikeit/\nSense and Sensibility\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/sensibility/\nThe Picture of Dorian Gray\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/doriangray/\nLord Jim\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/lordjim/\nThe Red and the Black\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/redblack/\nThe Comedy of Errors\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/errors/\nThe Tempest\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/tempest/\nUncle Vanya\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/drama/unclevanya/\nThe Portrait of a Lady\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/portraitlady/\nThe Last of the Mohicans\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/mohicans/\nDracula\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/dracula/\nJane Eyre\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/janeeyre/\nAdam Bede\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/adambede/\nThe Aeneid\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/aeneid/\nMain Street\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/mainstreet/\nOliver Twist\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/oliver/\nLittle Women\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/littlewomen/\nBabbitt\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/babbitt/\nSons and Lovers\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/sonsandlovers/\nThe House of Mirth\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/mirth/\nEmma\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/emma/\nPersuasion\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/persuasion/\nWuthering Heights\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/wuthering/\nMansfield Park\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/mansfieldpark/\nMiddlemarch\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/middlemarch/\nIvanhoe\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/ivanhoe/\nDavid Copperfield\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/copperfield/\nSister Carrie\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/sistercarrie/\nKidnapped\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/kidnapped/\nDr. Jekyll and Mr. Hyde\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/jekyll/\nThe Turn of the Screw\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/screw/\nCandide\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/candide/\nParadise Lost\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/poetry/paradiselost/\nNorthanger Abbey\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/northangerabbey/\nMy Antonia\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/antonia/\nHamlet\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/hamlet/\nOthello\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/othello/\nA Room with a View\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/room/\nCoriolanus\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/coriolanus/\nLooking Backward\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/lookingbackward/\nHeart of Darkness\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/heart-of-darkness/\nThe Power and the Glory\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/powerglory/\nThe Return of the Native\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/returnofnative/\nHowards End\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/howardsend/\nTypee\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/typee/\nAnne of Green Gables\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/anneofgreengables/\nRegeneration\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/regeneration/\nMaggie: A Girl of the Streets\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/maggie/\nThe Good Soldier\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/goodsoldier/\nThe Red Badge of Courage\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/redbadge/\nThe New Testament\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/newtestament/\nO Pioneers!\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/opioneers/\nHenry VIII\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/henryviii/\nThe Time Machine\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/timemachine/\nHenry IV Part 2\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/henry4pt2/\nMacbeth\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/macbeth/\nTroilus and Cressida\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/troilus/\nRichard II\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/richardii/\nHenry IV Part 1\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/henry4pt1/\nKing John\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/kingjohn/\nHenry VI Part 1\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/henry6pt1/\nRichard III\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/richardiii/\nTitus Andronicus\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/titus/\nJulius Caesar\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/juliuscaesar/\nLove's Labours Lost\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/labours/\nThe Two Gentlemen of Verona\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/twogentlemen/\nPygmalion\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/pygmalion/\nTwelfth Night\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/twelfthnight/\nThe Merry Wives of Windsor\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/merrywives/\nAn Enemy of the People\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/enemyofthepeople/\nMeasure for Measure\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/measure/\nAlice in Wonderland\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/alice/\nWhite Fang\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/fang/\nRomeo and Juliet\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/romeojuliet/\nThe Merchant of Venice\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/merchant/\nAll's Well That Ends Well\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/allswell/\nMajor Barbara\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/drama/majorbarbara/\nThe Jew of Malta\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/drama/jewofmalta/\nAn Ideal Husband\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/drama/idealhusband/\nThe Three Musketeers\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/3musk/\nCyrano de Bergerac\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/cyrano/\nThe Winter's Tale\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/winterstale/\nKing Lear\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/lear/\nBleak House\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/bleakhouse/\nThe Adventures of Huckleberry Finn\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/huckfinn/\nAnthem\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/anthem/\nEthan Frome\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/frome/\nThe Jungle\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/jungle/\nThe Age of Innocence\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/lit/ageofinnocence/\nTimon of Athens\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/timonofathens/\nMuch Ado About Nothing\thttps://web.archive.org/web/20210223161114/https://www.sparknotes.com/shakespeare/muchado/\n"
},
{
"path": "scripts/data_collection/thebestnotes/get_summaries.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\n\nimport pdb, os, csv, re, io, json\nimport urllib.request, urllib.error, urllib.parse\n\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom unidecode import unidecode\nimport argparse, string\n\nimport time\nfrom urllib.error import HTTPError, URLError\n\n# PARAMS\nSUMMARY_DIR = '../../raw_summaries/thebestnotes/summaries'\nMAIN_SITE = 'https://web.archive.org/web/20210111015641/http://thebestnotes.com/'\n\n# Summary list info\nsummary_list_file = 'literature_links.tsv.pruned'\n\n# Create a fresh file for the links that throw HTTP errors, so that we can try access them again\nf_errors = open(\"section_errors.txt\",\"w\")\n\n# Get contents of the summary file\nwith open(summary_list_file, 'r') as tsvfile:\n reader = csv.reader(tsvfile, delimiter='\\t')\n summary_infos = list(reader)\n\ndef unify_title(title):\n title_lower = title.lower().strip()\n title_clean = title_lower.translate(str.maketrans('', '', string.punctuation))\n return title_clean\n\ndef get_overview_paragraphs(overview):\n\n if 'https:/the' in overview:\n overview = overview.replace('https:/the', 'https://the')\n\n if 'http:/the' in overview:\n overview = overview.replace('http:/the', 'http://the')\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n except URLError as err:\n print (err, overview, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(overview), \"html.parser\")\n except Exception as e:\n print (overview, e)\n f_errors.write(overview + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n return\n except Exception as e:\n print (overview, e)\n f_errors.write(overview + \"\\t\" + str(e))\n return []\n\n overview_paragraphs = []\n flag = 0\n\n for paragraph in soup.findAll([\"p\", \"h2\", \"h6\"]):\n if 'synopsis' in \" \".join(paragraph.text.strip().lower().split()) and paragraph.name in [\"h2\", \"h6\"]:\n flag = 1\n continue\n\n # continue collecting text from the rest of the p tags\n if flag == 1 and \"thebestnotes\" not in paragraph.text.strip().lower() and (paragraph.name == 'p' or (paragraph.name in [\"h2\", \"h6\"] and paragraph.text.replace(\"\\r\\n\",\"\").strip() == \"\")):\n if paragraph.text.strip() != \"\":\n overview_paragraphs.append(unidecode(paragraph.text.replace(\"\\r\\n\",\"\").strip()))\n else:\n flag = 0\n # end collecting the summary when the above conditions are not met\n \n return overview_paragraphs\n\n\ndef get_section_paragraphs(section, section_titles, section_title_orig, specific_summary_dir, index):\n \n book_name = os.path.basename(specific_summary_dir)\n\n if 'https:/the' in section:\n section = section.replace('https:/the', 'https://the')\n\n if 'http:/the' in section:\n section = section.replace('http:/the', 'http://the')\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except URLError as err:\n print (err, section, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(section), \"html.parser\")\n except Exception as e:\n print (section, e)\n f_errors.write(section + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n return []\n except Exception as e:\n print (section_title_orig, section, e)\n f_errors.write(str(index+1) + \"\\t\" + section + \"\\t\" + str(section_titles) + \"\\t\" + section_title_orig + \"\\t\" + specific_summary_dir + \"\\n\")\n \n return []\n section_paragraphs = []\n flag = 0\n found = False\n\n # True if a structured page exists like - https://web.archive.org/web/20210111015641/http://thebestnotes.com/booknotes/Invisible_Man_Wells/The_Invisible_Man_Study_Guide14.html\n structured_page = 0\n\n if soup.findAll(\"div\", {\"class\": \"large-12 columns\"}) != []:\n structured_page = 1\n\n for section_title in section_titles:\n\n section_title = unify_title(section_title.strip())\n\n if structured_page:\n\n for paragraph in soup.findAll([\"p\", \"h2\", \"h6\"]):\n\n if section_title in unify_title(\" \".join(paragraph.text.strip().lower().split())) and paragraph.name in [\"h2\", \"h6\"]:\n flag = 1\n continue\n\n # continue collecting text from the rest of the p tags\n if flag == 1 and \"thebestnotes\" not in paragraph.text.strip().lower() and \\\n (paragraph.name == 'p' or\n (paragraph.name in [\"h2\", \"h6\"] and paragraph.text.strip() == \"\")):\n if paragraph.text.strip() != \"\":\n section_paragraphs.append(paragraph.text.replace(\"\\r\\n\",\"\").strip())\n else:\n flag = 0\n # end collecting the summary \n\n if (len(section_paragraphs) > 0):\n break\n \n else:\n\n for paragraph in soup.findAll([\"p\", \"h2\", \"h6\"]):\n\n if section_title in unify_title(\" \".join(paragraph.text.strip().lower().split())) and (paragraph.name in [\"h2\", \"h6\"] or (book_name == \"Looking Backward: 2000-1887\" and paragraph.name in [\"p\", \"h2\", \"h6\"])):\n flag = 1\n continue\n\n # continue collecting text from the rest of the p tags\n if flag == 1 and \"thebestnotes\" not in paragraph.text.strip().lower() and \\\n ((paragraph.name in [\"h2\", \"h6\"] or (book_name == \"Looking Backward: 2000-1887\" and paragraph.name in [\"p\", \"h2\", \"h6\"])) and (paragraph.text.replace(\"\\r\\n\",\"\").strip() == \"\" or \\\n paragraph.text.strip().lower() == \"summary\")):\n if paragraph.text.strip() != \"\":\n flag = 2\n continue\n else:\n continue\n\n if flag == 2 and \"thebestnotes\" not in paragraph.text.strip().lower() and (paragraph.name == 'p' or (paragraph.name in [\"h2\", \"h6\"] and paragraph.text.strip() == \"\")):\n if paragraph.text.strip() != \"\":\n section_paragraphs.append(paragraph.text.replace(\"\\r\\n\",\"\").strip())\n else:\n if flag == 2:\n flag = 0\n # end collecting the summary \n \n if (len(section_paragraphs) > 0):\n break\n # else there might be some text we missed\n\n return section_paragraphs\n\n\ndef save_section_para(section_paragraphs, section_titles, section_title_orig, section, specific_summary_dir, index):\n\n print (\"save_section_para: \", section_titles[0], section)\n\n section_text = \"\".join(section_paragraphs)\n\n section_dict = {}\n section_dict[\"name\"] = section_title_orig\n section_dict[\"summary\"] = section_text\n section_dict[\"analysis\"] = \"\"\n section_dict[\"url\"] = section\n\n output_fname = os.path.join(specific_summary_dir, 'section_%d.txt' % int(index))\n # print (\"output_fname: \", output_fname, \"\\n\")\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(section_dict, fp)\n\n# For each summary info\nfor k, (title, page_url) in enumerate(summary_infos):\n print('\\n>>> {}. {} - {} <<<'.format(k, title, page_url))\n\n\n # Create a directory for the work if needed\n specific_summary_dir = os.path.join(SUMMARY_DIR, title)\n if not os.path.exists(specific_summary_dir):\n os.makedirs(specific_summary_dir)\n else:\n print(\"Found existing directory\")\n # continue\n\n print (\"page_url: \", page_url)\n\n # Parse page\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except URLError as err:\n print (err, page_url, \"Retrying after sleep\")\n time.sleep(10)\n try:\n soup = BeautifulSoup(urllib.request.urlopen(page_url), \"html.parser\")\n except Exception as e:\n print (page_url, e)\n f_errors.write(page_url + \"\\t\" + str(e))\n f_errors.write(\"\\n\")\n continue\n\n one_level_up_url = os.path.dirname(page_url) + \"/\"\n\n #fetch section\n section_paragraphs = []\n\n navigation_links = soup.findAll(\"a\")\n\n index = -1 # For section numbers\n\n for link in navigation_links:\n\n if 'synopsis' in \" \".join(link.text.strip().lower().split()):\n overview = urllib.parse.urljoin(one_level_up_url, link.get(\"href\"))\n overview_title = link.text.strip().lower()\n print (\"overview: \",overview)\n\n overview_paragraphs = get_overview_paragraphs(overview)\n\n overview_text = \"\".join(overview_paragraphs)\n\n overview_dict = {}\n overview_dict[\"name\"] = \"overview\"\n overview_dict[\"summary\"] = overview_text\n overview_dict[\"analysis\"] = \"\"\n overview_dict[\"url\"] = overview\n\n output_fname = os.path.join(specific_summary_dir, \"overview.json\")\n with open(output_fname, 'w', encoding=\"utf-8\") as fp:\n json.dump(overview_dict, fp)\n\n else:\n\n section = urllib.parse.urljoin(one_level_up_url, link.get(\"href\"))\n\n section_title_orig = \" \".join(link.text.strip().lower().split())\n\n # Keep the original one first in the list of possible titles to match. For chapter numbers like \"TWENTY-THREE\" and \"TWENTY-FOUR\", which occur on the same web page.\n section_titles = [section_title_orig]\n\n # To handle cases where the og page says Chapter 1 - X, but the summary page just says X\n # Add the different kind of section titles we can have into a list\n if ('-' in section_title_orig):\n section_titles = section_titles + section_title_orig.strip().split('-')\n elif (':' in section_title_orig):\n section_titles = section_titles + section_title_orig.strip().split(':')\n\n if (section_title_orig == \"\"):\n continue\n\n section_paragraphs = get_section_paragraphs(section, section_titles, section_title_orig, specific_summary_dir, index)\n\n if (section_paragraphs != []):\n\n index += 1\n save_section_para(section_paragraphs, section_titles, section_title_orig, section, specific_summary_dir, index)\n"
},
{
"path": "scripts/data_collection/thebestnotes/get_works.py",
"content": "\"\"\"\n/*\n * Copyright (c) 2021, salesforce.com, inc.\n * All rights reserved.\n * SPDX-License-Identifier: BSD-3-Clause\n * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause\n */\n\"\"\"\n\nfrom builtins import zip, str, range\nimport pdb, os, csv, re, io, string\nimport urllib.request, urllib.error, urllib.parse\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\nfrom shutil import rmtree\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nimport time\n\n# PARAMS\nMAIN_SITE = 'https://web.archive.org/web/20210111015641/http://thebestnotes.com/'\n\nalphabet_list = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'QR', 'S', 'T', 'UV', 'W', 'XYZ']\n\nSEED_URL = 'https://web.archive.org/web/20210111015641/http://thebestnotes.com/list/titles'\n\nerrors_file = open(\"link_errors.txt\",\"w\")\n\ndef scrape_index_pages(seed_page):\n# For each summary info\n\n scraped_links = []\n\n for char in alphabet_list:\n books_page = seed_page + char + \".html\"\n\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n items = soup.findAll(\"div\", {\"class\": \"large-7 columns\"})\n books = items[0].findAll(\"p\")\n except Exception as e:\n time.sleep(10)\n \n # In order to handle timeouts\n try:\n soup = BeautifulSoup(urllib.request.urlopen(books_page), \"html.parser\")\n items = soup.findAll(\"div\", {\"class\": \"large-7 columns\"})\n books = items[0].findAll(\"p\")\n except Exception as e:\n print (\"Skipping: \", books_page, str(e))\n errors_file.write(books_page + \"\\t\" + str(e) + \"\\n\")\n continue\n\n # # # Go over each section\n for index, item in enumerate(books):\n # Parse section to get bullet point text\n \n try:\n item_title = item.find(\"a\").text\n item_url = item.find(\"a\").get(\"href\")\n\n print (\"item_title: \", \" \".join(item_title.split()))\n print (\"item_url: \", item_url.strip(), \"\\n\")\n\n #Don't add the book to the list if it isn't freely available\n if 'store' in item_url:\n continue\n\n scraped_links.append({\n \"title\": \" \".join(item_title.split()),\n \"url\": urllib.parse.urljoin(MAIN_SITE, item_url.strip())\n })\n \n except Exception as e:\n print (\"Skipping: \", str(item), str(e), \"\\n\")\n errors_file.write(str(item) + \"\\t\" + str(e) + \"\\n\")\n\n return scraped_links\n\n# generate literature links\nscraped_data = scrape_index_pages(SEED_URL)\n\nwith open(\"literature_links.tsv\", \"w\") as fd:\n for data in scraped_data:\n fd.write(\"%s\\t%s\\n\" % (data[\"title\"], data[\"url\"]))\n"
},
{
"path": "scripts/data_collection/thebestnotes/literature_links.tsv.pruned",
"content": "A Tale of Two Cities\thttps://web.archive.org/web/20190903025221/http://thebestnotes.com/booknotes/Tale_Of_Two_Cities_Dickens/A_Tale_Of_Two_Cities_Study_Guide01.html\nLittle Women\thttps://web.archive.org/web/20190903025235/http://thebestnotes.com/booknotes/Little_Women_Alcott/Little_Women_Study_Guide01.html\nPride and Prejudice\thttps://web.archive.org/web/20190903025227/http://thebestnotes.com/booknotes/Pride_And_Prejudice_Austen/Pride_And_Prejudice_Study_Guide01.html\nIvanhoe\thttps://web.archive.org/web/20190903025232/http://thebestnotes.com/booknotes/Ivanhoe_Sir_Walter_Scott/Ivanhoe_Study_Guide01.html\nTreasure Island\thttps://web.archive.org/web/20190903025221/http://thebestnotes.com/booknotes/Treasure_Island/Treasure_Island01.html\nAround the World in 80 Days\thttps://web.archive.org/web/20210417141554/http://thebestnotes.com/booknotes/Around_The_World_In_80_Days/Around_The_World_In_Eighty_Days01.html\nWhite Fang\thttps://web.archive.org/web/20190903025238/http://thebestnotes.com/booknotes/White_Fang/White_Fang01.html\nInvisible Man\thttps://web.archive.org/web/20190903025232/http://thebestnotes.com/booknotes/Invisible_Man_Ellison/Invisible_Man_Study_Guide01.html\nLooking Backward: 2000-1887\thttps://web.archive.org/web/20190903025235/http://thebestnotes.com/booknotes/Looking_Backward_Bellamy/Looking_Backward_Study_Guide01.html\n"
}
]
![](\"misc/book_sumv4.png\")