s and attributes since they are not needed anymore.\n# 2) Rewrite relative paths (\"/Brian-Bi\") to full URLs\n# 3) Download a copy of each embedded image\ndef cleanup_tree(doc, src, dest):\n for child in src.childNodes:\n if child.nodeType == Node.TEXT_NODE:\n # Text nodes can simply be left as-is\n dest.appendChild(child.cloneNode(False))\n continue\n if child.nodeType != Node.ELEMENT_NODE:\n # ???\n raise ValueError()\n # Otherwise, it's an element node.\n if child.tagName in ['br', 'hr']:\n dest.appendChild(child.cloneNode(False))\n elif child.tagName in ['b', 'i', 'u', 'h2', 'ol', 'ul', 'li', 'blockquote', 'wbr', 'p']:\n # This node doesn't need to be modified but its children might.\n # Also, we won't copy over any of its attributes.\n new_node = doc.createElement(child.tagName)\n cleanup_tree(doc, child, new_node)\n dest.appendChild(new_node)\n elif child.getAttribute('data-embed') != '':\n # This is a video. We want to copy the data-embed value, which is HTML for an iframe node.\n # So, we have to parse it into a separate document and import the node.\n iframe_html = child.getAttribute('data-embed')\n parser = HTMLParser(tree=treebuilders.getTreeBuilder('dom'))\n iframe_doc = parser.parse(iframe_html)\n try:\n iframe = iframe_doc.documentElement.childNodes[1].firstChild\n if iframe.tagName != 'iframe':\n raise ValueError()\n new_node = doc.importNode(iframe, False)\n # Quora uses a protocol-relative URL (//youtube.com/...) so let's make sure we rewrite this.\n src = new_node.getAttribute('src')\n if src.startswith('//'):\n new_node.setAttribute('src', 'http:' + src)\n # The video will look really bad if we don't explicitly set the dimensions.\n new_node.setAttribute('width', '525')\n new_node.setAttribute('height', '295')\n dest.appendChild(new_node)\n except Exception:\n print('[WARNING] Failed to parse video embed code', file=sys.stderr)\n # Bail out by just copying the original HTML\n dest.appendChild(child.cloneNode(True))\n elif child.tagName == 'code':\n # Inline code block. Strip the attributes.\n new_node = doc.createElement('code')\n dest.appendChild(new_node)\n cleanup_tree(doc, child, new_node)\n elif 'ContentFooter' in child.getAttribute('class') or 'hidden' in child.getAttribute('class'):\n # These are nodes we just want to skip.\n continue\n elif child.tagName in ['span', 'div']:\n # don't insert a span or div; just insert its contents\n cleanup_tree(doc, child, dest)\n elif child.tagName == 'a':\n # A link. We only want to copy the href, and pass the rest through.\n new_node = doc.createElement('a')\n href = child.getAttribute('href')\n if href.startswith('/'):\n href = 'http://quora.com' + href\n new_node.setAttribute('href', href)\n dest.appendChild(new_node)\n cleanup_tree(doc, child, new_node)\n elif child.tagName == 'img':\n src = child.getAttribute('master_src')\n if src == '':\n src = child.getAttribute('src')\n new_node = doc.createElement('img')\n new_node.setAttribute('src', src)\n new_node.setAttribute('alt', child.getAttribute('alt'))\n if args.no_download:\n dest.appendChild(new_node)\n continue\n # Save a copy of the image locally.\n # If an error occurs, just leave the src pointing to Quora.\n try:\n m = re.search('/([^/?]+)(\\?|$)', src)\n if m is None:\n raise ValueError()\n filename = m.group(1)\n if not filename.endswith('.png'):\n filename += '.png'\n try:\n img_fd = os.open(args.output_dir + '/' + filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)\n except OSError as error:\n if error.errno == errno.EEXIST:\n log_if_v('Image %s has already been saved; skipping' % filename)\n new_node.setAttribute('src', filename)\n continue\n else:\n raise\n log_if_v('Downloading image from %s' % src)\n closed = False\n try:\n img = urllib.request.urlopen(src).read()\n time.sleep(args.delay)\n os.write(img_fd, img)\n os.close(img_fd)\n closed = True\n except Exception:\n os.close(img_fd)\n closed = True\n try:\n os.remove(args.output_dir + '/' + filename)\n except:\n print('[WARNING] Failed to remove incomplete file %s' % filename, file=sys.stderr)\n raise\n finally:\n if not closed:\n os.close(img_fd)\n # Don't leave the file there; we will retry it next time.\n # If everything went according to plan, rewrite the src to the local file.\n new_node.setAttribute('src', filename)\n except urllib.error.URLError as error:\n print('[WARNING] Failed to download image from URL %s (%s)' % (src, error.reason), file=sys.stderr)\n except OSError as error:\n print('[WARNING] Failed to save image from URL %s to file %s (%s)' % (src, filename, error.strerror), file=sys.stderr)\n except ValueError:\n print('[WARNING] Failed to determine image name from URL %s' % src, file=sys.stderr)\n finally:\n dest.appendChild(new_node)\n elif child.tagName == 'pre':\n # Block (not inline) code. Quora's HTML already has the desired structure,\n # so we just need to strip the attributes from the .\n new_node = doc.createElement('pre')\n dest.appendChild(new_node)\n cleanup_tree(doc, child, new_node)\n else:\n print('[WARNING] Unrecognized node', file=sys.stderr)\n # Bail out by just copying the original HTML\n dest.appendChild(child.cloneNode(True))\n\nparser = argparse.ArgumentParser(description = 'Convert answers downloaded from Quora into a more portable HTML format')\nparser.add_argument('input_dir', nargs='?', default='./quora-answers', help='directory containing \"raw\" answers downloaded from Quora')\nparser.add_argument('output_dir', nargs='?', default='./quora-answers-cooked', help='where to store the images and converted answers')\nparser.add_argument('-d', '--delay', default=0, type=float, help='Time to sleep between downloads, in seconds')\nparser.add_argument('-n', '--no_download', action='store_true', help='Do not save images')\nparser.add_argument('-v', '--verbose', action='store_true', help='be verbose')\n\nglobal args\nargs = parser.parse_args()\n\n# Get a list of answers to convert...\nfilenames = list(filter(lambda f: f.endswith('.html'), os.listdir(args.input_dir)))\nfilenames.sort()\nif len(filenames) == 0:\n sys.exit('[FATAL] No .html files found in directory %s', args.input_dir)\nprint('Found %d answers' % len(filenames), file=sys.stderr)\n\nlog_if_v('Creating directory %s' % args.output_dir)\ntry:\n os.mkdir(args.output_dir, 0o700)\nexcept OSError as error:\n if error.errno == errno.EEXIST:\n log_if_v('Directory already exists')\n else:\n # This is the top level, and we have nothing else to do if we failed\n raise\n\nfor filename in filenames:\n sys.stderr.flush()\n print('Filename: ' + filename, file=sys.stderr)\n try:\n with open(args.input_dir + '/' + filename, 'rb') as page:\n page_html = page.read()\n except IOError as error:\n print('[ERROR] Failed to read %s (%s)' % (filename, error.strerror))\n continue\n\n # Get the HTML element containing just the answer itself.\n # Also get the title.\n parser = HTMLParser(tree=treebuilders.getTreeBuilder('dom'))\n document = parser.parse(page_html, default_encoding='utf-8')\n title_node = get_title_node(document) \n log_if_v('Title: ' + ('(could not be determined)' if title_node is None else get_text_content(title_node)))\n\n answer_node = None\n for node in document.getElementsByTagName('div'):\n if 'ExpandedAnswer' in node.getAttribute('class').split():\n answer_node = node\n break\n if answer_node is None:\n print('[WARNING] Failed to locate answer on page (filename: %s)' % filename, file=sys.stderr)\n continue\n\n # Construct our new page...\n new_page = document.createElement('html')\n head_node = document.createElement('head')\n if not title_node is None:\n head_node.appendChild(title_node)\n meta_node = document.createElement('meta')\n meta_node.setAttribute('charset', 'utf-8')\n head_node.appendChild(meta_node)\n css = (\"blockquote { border-left: 2px solid #ddd; color: #666; margin: 0; padding-left: 16px; } \"\n \"code, pre { background: #f4f4f4; } \"\n \"pre, h2 { margin: 0; } \"\n \"ul { margin: 0 0 0 16px; padding: 8px 0; } \"\n \"ol { margin: 0 0 0 28px; padding: 8px 0; } \"\n \"li { margin: 0 0 8px; } \")\n style_node = document.createElement('style')\n style_node.setAttribute('type', 'text/css')\n style_node.appendChild(document.createTextNode(css))\n head_node.appendChild(style_node)\n # Quora now uses MathJax, so set up the configuration object:\n script_node = document.createElement('script')\n script_text = document.createTextNode('window.MathJax = {\"showMathMenu\":false,\"messageStyle\":\"none\",\"errorSettings\":{\"style\":{\"color\":\"#000000\",\"font-style\":\"normal\"}},\"HTML-CSS\":{\"linebreaks\":{\"automatic\":true,\"width\":\"container\"},\"EqnChunk\":150,\"EqnChunkDelay\":20},\"tex2jax\":{\"inlineMath\":[[\"[math]\",\"[/math]\"]],\"displayMath\":[],\"ignoreClass\":\"edit_latex|qtext_editor_content|ignore_latex\",\"processClass\":\"render_latex\",\"processEnvironments\":false,\"preview\":\"none\"},\"TeX\":{\"noUndefined\":{\"attributes\":{\"mathcolor\":\"red\"}},\"noErrors\":{\"multiLine\":true,\"style\":{\"max-width\":\"100%\",\"overflow\":\"hidden\"}},\"Macros\":{\"C\":\"{\\\\mathbb{C}}\",\"N\":\"{\\\\mathbb{N}}\",\"O\":\"{\\\\emptyset}\",\"Q\":\"{\\\\mathbb{Q}}\",\"R\":\"{\\\\mathbb{R}}\",\"Z\":\"{\\\\mathbb{Z}}\"}},\"fast-preview\":{\"disabled\":true},\"Safe\":{\"allow\":{\"URLs\":\"none\",\"classes\":\"none\",\"cssIDs\":\"none\",\"styles\":\"none\",\"fontsize\":\"none\",\"require\":\"none\"}}};')\n script_node.appendChild(script_text)\n head_node.appendChild(script_node)\n # and then load MathJax:\n script_node = document.createElement('script')\n script_node.setAttribute('type', 'text/javascript')\n script_node.setAttribute('src', 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_HTMLorMML,Safe')\n head_node.appendChild(script_node)\n new_page.appendChild(head_node)\n body_node = document.createElement('body')\n # This step processes Quora's HTML into a more lightweight and portable form.\n cleanup_tree(document, answer_node, body_node)\n new_page.appendChild(body_node)\n # Okay! Finally, save the HTML.\n walker = treewalkers.getTreeWalker('dom')(new_page)\n try:\n with open(args.output_dir + '/' + filename, 'wb', 0o600) as saved_page:\n saved_page.write(b'')\n saved_page.write(serializer.serialize(new_page, 'dom', 'utf-8', omit_optional_tags=False))\n except IOError as error:\n print('[ERROR] Failed to save to file %s (%s)' % (filename, error.strerror), file=sys.stderr)\n\nprint('Done', file=sys.stderr)\n\n"
},
{
"path": "crawler.py",
"content": "#!/usr/bin/env python3\nimport argparse\nimport errno\nimport json\nimport os\nimport re\nimport sys\nimport time\nimport urllib.error\nimport urllib.request\n\ndef log_if_v(msg):\n if args.verbose:\n print('[DEBUG] %s' % msg, file=sys.stderr)\n\n# Given origin (timestamp offset by time zone) and string from Quora, e.g.\n# \"Added 31 Jan\", returns a string such as '2015-01-31'.\n# Quora's short date strings don't provide enough information to determine the\n# exact time, unless it was within the last day, so we won't bother to be any\n# more precise.\ndef parse_quora_date(origin, quora_str):\n days_of_week = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']\n months_of_year = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']\n _, _, date_str = quora_str.partition('Added ')\n date_str = date_str.strip()\n if date_str == '':\n raise ValueError('\"%s\" does not appear to indicate when answer was added' % quora_str)\n m0 = re.match('just now$', date_str)\n m1 = re.match('(\\d+)m ago$', date_str)\n m2 = re.match('(\\d+)h ago$', date_str)\n m3 = re.match('(' + '|'.join(days_of_week) + ')$', date_str)\n m4 = re.match('(' + '|'.join(months_of_year) + ') (\\d+)$', date_str)\n m5 = re.match('(' + '|'.join(months_of_year) + ') (\\d+), (\\d+)$', date_str)\n m6 = re.match('(\\d+)[ap]m$', date_str)\n if not m0 is None or not m6 is None:\n # Using origin for time in am / pm since the time of the day will be discarded anyway\n tm = time.gmtime(origin)\n elif not m1 is None:\n tm = time.gmtime(origin - 60*int(m1.group(1)))\n elif not m2 is None:\n tm = time.gmtime(origin - 3600*int(m2.group(1)))\n elif not m3 is None:\n # Walk backward until we reach the given day of the week\n day_of_week = days_of_week.index(m3.group(1))\n offset = 1\n while offset <= 7:\n tm = time.gmtime(origin - 86400*offset)\n if tm.tm_wday == day_of_week:\n break\n offset += 1\n else:\n raise ValueError('date \"%s\" is invalid' % date_str)\n elif not m4 is None:\n # Walk backward until we reach the given month and year\n month_of_year = months_of_year.index(m4.group(1)) + 1\n day_of_month = int(m4.group(2))\n offset = 1\n while offset <= 366:\n tm = time.gmtime(origin - 86400*offset)\n if tm.tm_mon == month_of_year and tm.tm_mday == day_of_month:\n break\n offset += 1\n else:\n raise ValueError('date \"%s\" is invalid' % date_str)\n elif not m5 is None:\n # may raise ValueError\n tm = time.strptime(date_str, '%b %d, %Y')\n else:\n raise ValueError('date \"%s\" could not be interpreted' % date_str)\n return '%d-%02d-%02d' % (tm.tm_year, tm.tm_mon, tm.tm_mday)\n\nparser = argparse.ArgumentParser(description = 'Download a set of answers from Quora')\nparser.add_argument('input_file', help='file containing JSON-encoded list of timestamped URLs to download')\nparser.add_argument('output_dir', nargs='?', default='./quora-answers', help='where to store the downloaded answers and images')\nparser.add_argument('-d', '--delay', default=0, type=float, help='Time to sleep between answers, in seconds')\nparser.add_argument('-t', '--origin_timestamp', default=None, type=int, help='JS time when the list of URLs was fetched')\nparser.add_argument('-z', '--origin_timezone', default=None, type=int, help='browser timezone')\nparser.add_argument('-v', '--verbose', action='store_true', help='enable debug messages')\nparser.add_argument('-o', '--overwrite', action='store_true', help='Overwrite existing answers')\n\nglobal args\nargs = parser.parse_args()\n\n# Determine the origin for relative date computation\nif args.origin_timestamp is None:\n log_if_v('Using current time')\n args.origin_timestamp = time.time()\nelse:\n args.origin_timestamp //= 1000\nif args.origin_timezone is None:\n log_if_v('Using system time zone')\n args.origin_timezone = time.timezone\nelse:\n args.origin_timezone *= 60\norigin = args.origin_timestamp - args.origin_timezone\n\n# Load the list of answer URLs from the input file.\nlog_if_v('Loading input file %s' % args.input_file)\nwith open(args.input_file, 'rb') as input_file:\n answers = json.load(input_file)\nprint('Found %d answers' % len(answers), file=sys.stderr)\n\n# Check the validity of the input\nif type(answers) != list:\n sys.exit('[FATAL] Incorrect input format')\nfor e in answers:\n if type(e) != list or len(e) != 2 or type(e[0]) != str or type(e[1]) != str:\n sys.exit('[FATAL] Incorrect input format')\n\nlog_if_v('Creating directory %s' % args.output_dir)\ntry:\n os.mkdir(args.output_dir, 0o700)\nexcept OSError as error:\n if error.errno == errno.EEXIST:\n log_if_v('Directory already exists')\n else:\n # This is the top level, and we have nothing else to do if we failed\n raise\nos.chdir(args.output_dir)\ndownload_file_count = 0\nfor e in answers:\n sys.stderr.flush()\n url = e[0]\n print('URL: %s' % url, file=sys.stderr)\n\n # Determine the date when this answer was written\n try:\n added_time = parse_quora_date(origin, e[1])\n except ValueError as error:\n print('[WARNING] Failed to parse date: %s' % str(error), file=sys.stderr)\n added_time = 'xxxx-xx-xx'\n print('Date: %s' % added_time, file=sys.stderr)\n\n # Get the part of the URL indicating the question title; we will save under this name\n m1 = re.search('quora\\.com/([^/]+)/answer', url)\n # if there's a context topic\n m2 = re.search('quora\\.com/[^/]+/([^/]+)/answer', url)\n filename = added_time + ' '\n if not m1 is None:\n filename += m1.group(1)\n elif not m2 is None:\n filename += m2.group(1)\n else:\n print('[ERROR] Could not find question part of URL %s; skipping' % url, file=sys.stderr)\n continue\n # Trim the filename if it's too long. 255 bytes is the limit on many filesystems.\n total_byte_length = len(bytes(filename + '.html', encoding=\"utf-8\"))\n filename_bytes = bytes(filename, encoding=\"utf-8\")\n if total_byte_length > 255:\n filename_bytes = filename_bytes[:255-total_byte_length]\n well_formed = False\n while not well_formed:\n try:\n filename = str(filename_bytes, encoding=\"utf-8\")\n well_formed = True\n except UnicodeDecodeError:\n filename_bytes = filename_bytes[:-1]\n log_if_v('Filename was truncated to at most 255 bytes.')\n filename += '.html'\n log_if_v('Filename: %s' % filename)\n\n # If overwrite is enabled or the answer doesn't exist\n if args.overwrite or not os.path.isfile(filename):\n # Fetch the URL to find the answer\n log_if_v('Downloading answer from URL %s' % url)\n try:\n page_html = urllib.request.urlopen(url).read()\n with open(filename, 'wb') as f:\n f.write(page_html)\n except urllib.error.URLError as error:\n print('[ERROR] Failed to download answer from URL %s (%s)' % (url, error.reason), file=sys.stderr)\n continue\n except IOError as error:\n print('[ERROR] Failed to save answer to file %s (%s)' % (filename, error.strerror), file=sys.stderr)\n\n download_file_count += 1\n time.sleep(args.delay)\n else:\n log_if_v('Answer File : %s Already Exists. Skipping' % filename)\n\nprint('Done. Downloaded %d files' % download_file_count, file=sys.stderr)\n"
}
]