[ { "path": ".github/CODEOWNERS", "content": "# Codeowners for these exercise files:\n# * (asterisk) deotes \"all files and folders\"\n# Example: * @producer @instructor\n" }, { "path": ".github/ISSUE_TEMPLATE.md", "content": "\n\n## Issue Overview\n\n\n## Describe your environment\n\n\n## Steps to Reproduce\n\n1.\n2.\n3.\n4.\n\n## Expected Behavior\n\n\n## Current Behavior\n\n\n## Possible Solution\n\n\n## Screenshots / Video\n\n\n## Related Issues\n\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "\n" }, { "path": ".github/workflows/main.yml", "content": "name: Copy To Branches\non:\n workflow_dispatch:\njobs:\n copy-to-branches:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n with:\n fetch-depth: 0\n - name: Copy To Branches Action\n uses: planetoftheweb/copy-to-branches@v1\n" }, { "path": ".gitignore", "content": ".DS_Store\nnode_modules\n.tmp\nnpm-debug.log\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/__init__.py", "content": "" }, { "path": "01_03/ietf_scraper/ietf_scraper/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass IetfScraperItem(scrapy.Item):\n # define the fields for your item here like:\n # name = scrapy.Field()\n pass\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass IetfScraperSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass IetfScraperDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass IetfScraperPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for ietf_scraper project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'ietf_scraper'\n\nSPIDER_MODULES = ['ietf_scraper.spiders']\nNEWSPIDER_MODULE = 'ietf_scraper.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'ietf_scraper (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'ietf_scraper.pipelines.IetfScraperPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "01_03/ietf_scraper/ietf_scraper/spiders/ietf.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\n\n\nclass IetfSpider(scrapy.Spider):\n name = 'ietf'\n allowed_domains = ['pythonscraping.com']\n start_urls = ['http://pythonscraping.com/linkedin/ietf.html']\n\n def parse(self, response):\n return {'title': response.xpath('//span[@class=\"title\"]/text()').get()}\n" }, { "path": "01_03/ietf_scraper/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = ietf_scraper.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = ietf_scraper\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/__init__.py", "content": "" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass IetfScraperItem(scrapy.Item):\n # define the fields for your item here like:\n # name = scrapy.Field()\n pass\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass IetfScraperSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass IetfScraperDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass IetfScraperPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for ietf_scraper project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'ietf_scraper'\n\nSPIDER_MODULES = ['ietf_scraper.spiders']\nNEWSPIDER_MODULE = 'ietf_scraper.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'ietf_scraper (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'ietf_scraper.pipelines.IetfScraperPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "01_04_b/ietf_scraper/ietf_scraper/spiders/ietf.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\n\n\nclass IetfSpider(scrapy.Spider):\n name = 'ietf'\n allowed_domains = ['pythonscraping.com']\n start_urls = ['http://pythonscraping.com/linkedin/ietf.html']\n\n def parse(self, response):\n #title = response.css('span.title::text').get()\n title = response.xpath('//span[@class=\"title\"]/text()').get()\n return {\"title\": title}\n" }, { "path": "01_04_b/ietf_scraper/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = ietf_scraper.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = ietf_scraper\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/__init__.py", "content": "" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass IetfScraperItem(scrapy.Item):\n # define the fields for your item here like:\n # name = scrapy.Field()\n pass\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass IetfScraperSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass IetfScraperDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass IetfScraperPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for ietf_scraper project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'ietf_scraper'\n\nSPIDER_MODULES = ['ietf_scraper.spiders']\nNEWSPIDER_MODULE = 'ietf_scraper.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'ietf_scraper (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'ietf_scraper.middlewares.IetfScraperDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'ietf_scraper.pipelines.IetfScraperPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "01_04_e/ietf_scraper/ietf_scraper/spiders/ietf.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nimport w3lib.html\n\nclass IetfSpider(scrapy.Spider):\n name = 'ietf'\n allowed_domains = ['pythonscraping.com']\n start_urls = ['http://pythonscraping.com/linkedin/ietf.html']\n\n def parse(self, response):\n return {\n 'number': response.xpath('//span[@class=\"rfc-no\"]/text()').get(),\n 'title': response.xpath('//meta[@name=\"DC.Title\"]/@content').get(),\n # 'title': response.xpath('//span[@class=\"title\"]/text()').get(),\n 'date': response.xpath('//span[@class=\"date\"]/text()').get(),\n # 'date': response.xpath('//meta[@name=\"DC.Date.Issued\"]/@content').get(),\n 'description': response.xpath('//meta[@name=\"DC.Description.Abstract\"]/@content').get(),\n 'author': response.xpath('//meta[@name=\"DC.Creator\"]/@content').get(),\n # 'author': response.xpath('//span[@class=\"author-name\"]/text()').get(),\n 'company': response.xpath('//span[@class=\"author-company\"]/text()').get(),\n 'address': response.xpath('//span[@class=\"address\"]/text()').get(),\n 'text': w3lib.html.remove_tags(response.xpath('//div[@class=\"text\"]').get()),\n 'headings': response.xpath('//span[@class=\"subheading\"]/text()').getall()\n }\n" }, { "path": "01_04_e/ietf_scraper/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = ietf_scraper.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = ietf_scraper\n" }, { "path": "02_01/article_scraper/article_scraper/__init__.py", "content": "" }, { "path": "02_01/article_scraper/article_scraper/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass ArticleScraperItem(scrapy.Item):\n # define the fields for your item here like:\n # name = scrapy.Field()\n pass\n" }, { "path": "02_01/article_scraper/article_scraper/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleScraperSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleScraperDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_01/article_scraper/article_scraper/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleScraperPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_01/article_scraper/article_scraper/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_scraper project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_scraper'\n\nSPIDER_MODULES = ['article_scraper.spiders']\nNEWSPIDER_MODULE = 'article_scraper.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_scraper (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_scraper.middlewares.ArticleScraperSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_scraper.middlewares.ArticleScraperDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_scraper.pipelines.ArticleScraperPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_01/article_scraper/article_scraper/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_01/article_scraper/article_scraper/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n rules = [Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)]\n\n def parse_info(self, response):\n return {\n 'title': response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()').get(),\n 'url': response.url,\n 'last_edited': response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n }\n" }, { "path": "02_01/article_scraper/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_scraper.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_scraper\n" }, { "path": "02_02_b/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_02_b/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass ArticleCrawlerItem(scrapy.Item):\n # define the fields for your item here like:\n # name = scrapy.Field()\n pass\n" }, { "path": "02_02_b/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_02_b/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleCrawlerPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_02_b/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_crawler.pipelines.ArticleCrawlerPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_02_b/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_02_b/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n def parse_info(self, response):\n return {\n \"title\": response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()'),\n \"url\": response.url,\n \"last_edited\": response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n }\n" }, { "path": "02_02_b/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_02_e/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_02_e/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass Article(scrapy.Item):\n title = scrapy.Field()\n url = scrapy.Field()\n lastUpdated = scrapy.Field()\n\n" }, { "path": "02_02_e/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_02_e/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleCrawlerPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_02_e/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_crawler.pipelines.ArticleCrawlerPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_02_e/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_02_e/article_crawler/article_crawler/spiders/articles.csv", "content": "lastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 26 May 2020, at 18:28\",Obie Award,https://en.wikipedia.org/wiki/Obie_Award\r\n\" This page was last edited on 5 October 2020, at 13:16\",Richard Dean Anderson,https://en.wikipedia.org/wiki/Richard_Dean_Anderson\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n" }, { "path": "02_02_e/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom article_crawler.items import Article\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n def parse_info(self, response):\n article = Article()\n article['title']= response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()')\n article['url'] = response.url\n\n article['lastUpdated'] = response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n return article\n" }, { "path": "02_02_e/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_03_b/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_03_b/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass Article(scrapy.Item):\n title = scrapy.Field()\n url = scrapy.Field()\n lastUpdated = scrapy.Field()\n\n" }, { "path": "02_03_b/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_03_b/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleCrawlerPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_03_b/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_crawler.pipelines.ArticleCrawlerPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_03_b/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_03_b/article_crawler/article_crawler/spiders/articles.csv", "content": "lastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 26 May 2020, at 18:28\",Obie Award,https://en.wikipedia.org/wiki/Obie_Award\r\n\" This page was last edited on 5 October 2020, at 13:16\",Richard Dean Anderson,https://en.wikipedia.org/wiki/Richard_Dean_Anderson\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n" }, { "path": "02_03_b/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom article_crawler.items import Article\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n def parse_info(self, response):\n article = Article()\n article['title']= response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()')\n article['url'] = response.url\n\n article['lastUpdated'] = response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n return article\n" }, { "path": "02_03_b/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_03_e/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_03_e/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass Article(scrapy.Item):\n title = scrapy.Field()\n url = scrapy.Field()\n lastUpdated = scrapy.Field()\n\n" }, { "path": "02_03_e/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_03_e/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleCrawlerPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_03_e/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nCLOSESPIDER_PAGECOUNT=10\n\nFEED_URI='articles.json'\nFEED_FORMAT='json'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_crawler.pipelines.ArticleCrawlerPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_03_e/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_03_e/article_crawler/article_crawler/spiders/articles.csv", "content": "lastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 26 May 2020, at 18:28\",Obie Award,https://en.wikipedia.org/wiki/Obie_Award\r\n\" This page was last edited on 5 October 2020, at 13:16\",Richard Dean Anderson,https://en.wikipedia.org/wiki/Richard_Dean_Anderson\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 25 September 2020, at 05:22\",\"[]\",https://en.wikipedia.org/wiki/She%27s_Having_a_Baby\r\n\" This page was last edited on 2 July 2020, at 13:53\",SNAC,https://en.wikipedia.org/wiki/SNAC\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 20 September 2020, at 07:23\",[],https://en.wikipedia.org/wiki/Los_Angeles_Daily_News\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 22 July 2020, at 18:39\", (2020 TV series),https://en.wikipedia.org/wiki/Ana_(2020_TV_series)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 6 June 2020, at 20:53\",Bruce Gilbert,https://en.wikipedia.org/wiki/Bruce_Gilbert\r\n\" This page was last edited on 23 June 2020, at 19:06\", (TV series),https://en.wikipedia.org/wiki/The_Remix_(TV_series)\r\n\" This page was last edited on 6 October 2020, at 13:04\",[],https://en.wikipedia.org/wiki/The_New_York_Times\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 11 May 2020, at 14:47\",National Library of Latvia,https://en.wikipedia.org/wiki/National_Library_of_Latvia\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 5 October 2020, at 15:12\",Judy Garland,https://en.wikipedia.org/wiki/Judy_Garland\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 6 October 2020, at 03:55\",IMDb,https://en.wikipedia.org/wiki/IMDb\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 30 July 2020, at 18:19\",Virtual International Authority File,https://en.wikipedia.org/wiki/VIAF_(identifier)\r\n\" This page was last edited on 18 September 2020, at 03:04\",Trove,https://en.wikipedia.org/wiki/Trove\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n" }, { "path": "02_03_e/article_crawler/article_crawler/spiders/articles.json", "content": "[\n{\"title\": \"Kevin Bacon\", \"url\": \"https://en.wikipedia.org/wiki/Kevin_Bacon\", \"lastUpdated\": \" This page was last edited on 19 September 2020, at 00:35\"},\n{\"title\": \"Fox Broadcasting Company\", \"url\": \"https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\", \"lastUpdated\": \" This page was last edited on 6 October 2020, at 15:27\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Patriots_Day_(film)\", \"lastUpdated\": \" This page was last edited on 23 September 2020, at 20:01\"},\n{\"title\": \" (TV series)\", \"url\": \"https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\", \"lastUpdated\": \" This page was last edited on 18 August 2020, at 20:30\"},\n{\"title\": \"Screen Actors Guild Awards\", \"url\": \"https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\", \"lastUpdated\": \" This page was last edited on 21 July 2020, at 00:07\"},\n,\n,\n{\"title\": \"Primetime Emmy Award\", \"url\": \"https://en.wikipedia.org/wiki/Primetime_Emmy_Award\", \"lastUpdated\": \" This page was last edited on 22 September 2020, at 10:27\"},\n{\"title\": \"Golden Globe Awards\", \"url\": \"https://en.wikipedia.org/wiki/Golden_Globe_Award\", \"lastUpdated\": \" This page was last edited on 8 September 2020, at 12:45\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Black_Mass_(film)\", \"lastUpdated\": \" This page was last edited on 1 October 2020, at 17:17\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Frost/Nixon_(film)\", \"lastUpdated\": \" This page was last edited on 16 August 2020, at 00:08\"},\n,\n,\n{\"title\": \"HBO\", \"url\": \"https://en.wikipedia.org/wiki/HBO\", \"lastUpdated\": \" This page was last edited on 7 October 2020, at 00:10\"},\n,\n{\"title\": \"Circle in the Square Theatre\", \"url\": \"https://en.wikipedia.org/wiki/Circle_in_the_Square\", \"lastUpdated\": \" This page was last edited on 27 September 2020, at 21:06\"},\n{\"title\": \"Main Page\", \"url\": \"https://en.wikipedia.org/wiki/Main_Page\", \"lastUpdated\": \" This page was last edited on 23 July 2020, at 12:44\"},\n{\"title\": \"WorldCat\", \"url\": \"https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\", \"lastUpdated\": \" This page was last edited on 3 October 2020, at 11:46\"},\n{\"title\": \"Virtual International Authority File\", \"url\": \"https://en.wikipedia.org/wiki/VIAF_(identifier)\", \"lastUpdated\": \" This page was last edited on 30 July 2020, at 18:19\"},\n{\"title\": \"Trove\", \"url\": \"https://en.wikipedia.org/wiki/Trove\", \"lastUpdated\": \" This page was last edited on 18 September 2020, at 03:04\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Wild_Things_(film)\", \"lastUpdated\": \" This page was last edited on 29 September 2020, at 08:35\"},\n{\"title\": \"Syst\\u00e8me universitaire de documentation\", \"url\": \"https://en.wikipedia.org/wiki/SUDOC_(identifier)\", \"lastUpdated\": \" This page was last edited on 19 October 2019, at 13:42\"},\n{\"title\": \"SNAC\", \"url\": \"https://en.wikipedia.org/wiki/SNAC\", \"lastUpdated\": \" This page was last edited on 2 July 2020, at 13:53\"}\n]" }, { "path": "02_03_e/article_crawler/article_crawler/spiders/articles.xml", "content": "\n\nKevin Baconhttps://en.wikipedia.org/wiki/Kevin_Bacon This page was last edited on 19 September 2020, at 00:35\n (TV series)https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series) This page was last edited on 18 August 2020, at 20:30\nPrimetime Emmy Awardhttps://en.wikipedia.org/wiki/Primetime_Emmy_Award This page was last edited on 22 September 2020, at 10:27\nSixDegrees.orghttps://en.wikipedia.org/wiki/SixDegrees.org This page was last edited on 20 March 2020, at 11:35\nGolden Globe Award for Best Actor – Television Series Musical or Comedyhttps://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy This page was last edited on 14 August 2020, at 04:30\nList of social networking websiteshttps://en.wikipedia.org/wiki/Social_networks This page was last edited on 6 September 2020, at 23:58\nSix Degrees of Kevin Baconhttps://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon This page was last edited on 2 October 2020, at 20:10\nScreen Actors Guild Awardshttps://en.wikipedia.org/wiki/Screen_Actors_Guild_Award This page was last edited on 21 July 2020, at 00:07\n<value><Selector xpath='//h1/i/text()' data='The Guardian'></value>https://en.wikipedia.org/wiki/The_Guardian This page was last edited on 18 September 2020, at 16:08\nHollywood Walk of Famehttps://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame This page was last edited on 3 October 2020, at 12:56\nAcademy Awardshttps://en.wikipedia.org/wiki/Academy_Award This page was last edited on 1 October 2020, at 12:55\nCannes Film Festivalhttps://en.wikipedia.org/wiki/Cannes_Film_Festival This page was last edited on 5 October 2020, at 12:51\n<value><Selector xpath='//h1/i/text()' data='Taking Chance'></value>https://en.wikipedia.org/wiki/Taking_Chance This page was last edited on 3 September 2020, at 14:05\nAlan Rickmanhttps://en.wikipedia.org/wiki/Alan_Rickman This page was last edited on 7 October 2020, at 00:12\n<value><Selector xpath='//h1/i/text()' data='The Following'></value>https://en.wikipedia.org/wiki/The_Following This page was last edited on 11 September 2020, at 16:17\nMain Pagehttps://en.wikipedia.org/wiki/Main_Page This page was last edited on 23 July 2020, at 12:44\nFox Broadcasting Companyhttps://en.wikipedia.org/wiki/Fox_Broadcasting_Company This page was last edited on 6 October 2020, at 15:27\nGolden Globe Awardshttps://en.wikipedia.org/wiki/Golden_Globe_Award This page was last edited on 8 September 2020, at 12:45\nWorldCathttps://en.wikipedia.org/wiki/WorldCat_Identities_(identifier) This page was last edited on 3 October 2020, at 11:46\nVirtual International Authority Filehttps://en.wikipedia.org/wiki/VIAF_(identifier) This page was last edited on 30 July 2020, at 18:19\nHBOhttps://en.wikipedia.org/wiki/HBO This page was last edited on 7 October 2020, at 00:10\nTrovehttps://en.wikipedia.org/wiki/Trove This page was last edited on 18 September 2020, at 03:04\nSystème universitaire de documentationhttps://en.wikipedia.org/wiki/SUDOC_(identifier) This page was last edited on 19 October 2019, at 13:42\n" }, { "path": "02_03_e/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom article_crawler.items import Article\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n custom_settings={\n 'FEED_URI': 'articles.xml',\n 'FEED_FORMAT': 'xml'\n }\n\n def parse_info(self, response):\n article = Article()\n article['title']= response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()')\n article['url'] = response.url\n\n article['lastUpdated'] = response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n return article\n" }, { "path": "02_03_e/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_04_b/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_04_b/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass Article(scrapy.Item):\n title = scrapy.Field()\n url = scrapy.Field()\n lastUpdated = scrapy.Field()\n\n" }, { "path": "02_04_b/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_04_b/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\n\nclass ArticleCrawlerPipeline:\n def process_item(self, item, spider):\n return item\n" }, { "path": "02_04_b/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nCLOSESPIDER_PAGECOUNT=10\n\nFEED_URI='articles.json'\nFEED_FORMAT='json'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'article_crawler.pipelines.ArticleCrawlerPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_04_b/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_04_b/article_crawler/article_crawler/spiders/articles.csv", "content": "lastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 26 May 2020, at 18:28\",Obie Award,https://en.wikipedia.org/wiki/Obie_Award\r\n\" This page was last edited on 5 October 2020, at 13:16\",Richard Dean Anderson,https://en.wikipedia.org/wiki/Richard_Dean_Anderson\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 25 September 2020, at 05:22\",\"[]\",https://en.wikipedia.org/wiki/She%27s_Having_a_Baby\r\n\" This page was last edited on 2 July 2020, at 13:53\",SNAC,https://en.wikipedia.org/wiki/SNAC\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 20 September 2020, at 07:23\",[],https://en.wikipedia.org/wiki/Los_Angeles_Daily_News\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 22 July 2020, at 18:39\", (2020 TV series),https://en.wikipedia.org/wiki/Ana_(2020_TV_series)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 6 June 2020, at 20:53\",Bruce Gilbert,https://en.wikipedia.org/wiki/Bruce_Gilbert\r\n\" This page was last edited on 23 June 2020, at 19:06\", (TV series),https://en.wikipedia.org/wiki/The_Remix_(TV_series)\r\n\" This page was last edited on 6 October 2020, at 13:04\",[],https://en.wikipedia.org/wiki/The_New_York_Times\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 11 May 2020, at 14:47\",National Library of Latvia,https://en.wikipedia.org/wiki/National_Library_of_Latvia\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 5 October 2020, at 15:12\",Judy Garland,https://en.wikipedia.org/wiki/Judy_Garland\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 6 October 2020, at 03:55\",IMDb,https://en.wikipedia.org/wiki/IMDb\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 30 July 2020, at 18:19\",Virtual International Authority File,https://en.wikipedia.org/wiki/VIAF_(identifier)\r\n\" This page was last edited on 18 September 2020, at 03:04\",Trove,https://en.wikipedia.org/wiki/Trove\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n" }, { "path": "02_04_b/article_crawler/article_crawler/spiders/articles.json", "content": "[\n{\"title\": \"Kevin Bacon\", \"url\": \"https://en.wikipedia.org/wiki/Kevin_Bacon\", \"lastUpdated\": \" This page was last edited on 19 September 2020, at 00:35\"},\n{\"title\": \"Fox Broadcasting Company\", \"url\": \"https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\", \"lastUpdated\": \" This page was last edited on 6 October 2020, at 15:27\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Patriots_Day_(film)\", \"lastUpdated\": \" This page was last edited on 23 September 2020, at 20:01\"},\n{\"title\": \" (TV series)\", \"url\": \"https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\", \"lastUpdated\": \" This page was last edited on 18 August 2020, at 20:30\"},\n{\"title\": \"Screen Actors Guild Awards\", \"url\": \"https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\", \"lastUpdated\": \" This page was last edited on 21 July 2020, at 00:07\"},\n,\n,\n{\"title\": \"Primetime Emmy Award\", \"url\": \"https://en.wikipedia.org/wiki/Primetime_Emmy_Award\", \"lastUpdated\": \" This page was last edited on 22 September 2020, at 10:27\"},\n{\"title\": \"Golden Globe Awards\", \"url\": \"https://en.wikipedia.org/wiki/Golden_Globe_Award\", \"lastUpdated\": \" This page was last edited on 8 September 2020, at 12:45\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Black_Mass_(film)\", \"lastUpdated\": \" This page was last edited on 1 October 2020, at 17:17\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Frost/Nixon_(film)\", \"lastUpdated\": \" This page was last edited on 16 August 2020, at 00:08\"},\n,\n,\n{\"title\": \"HBO\", \"url\": \"https://en.wikipedia.org/wiki/HBO\", \"lastUpdated\": \" This page was last edited on 7 October 2020, at 00:10\"},\n,\n{\"title\": \"Circle in the Square Theatre\", \"url\": \"https://en.wikipedia.org/wiki/Circle_in_the_Square\", \"lastUpdated\": \" This page was last edited on 27 September 2020, at 21:06\"},\n{\"title\": \"Main Page\", \"url\": \"https://en.wikipedia.org/wiki/Main_Page\", \"lastUpdated\": \" This page was last edited on 23 July 2020, at 12:44\"},\n{\"title\": \"WorldCat\", \"url\": \"https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\", \"lastUpdated\": \" This page was last edited on 3 October 2020, at 11:46\"},\n{\"title\": \"Virtual International Authority File\", \"url\": \"https://en.wikipedia.org/wiki/VIAF_(identifier)\", \"lastUpdated\": \" This page was last edited on 30 July 2020, at 18:19\"},\n{\"title\": \"Trove\", \"url\": \"https://en.wikipedia.org/wiki/Trove\", \"lastUpdated\": \" This page was last edited on 18 September 2020, at 03:04\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Wild_Things_(film)\", \"lastUpdated\": \" This page was last edited on 29 September 2020, at 08:35\"},\n{\"title\": \"Syst\\u00e8me universitaire de documentation\", \"url\": \"https://en.wikipedia.org/wiki/SUDOC_(identifier)\", \"lastUpdated\": \" This page was last edited on 19 October 2019, at 13:42\"},\n{\"title\": \"SNAC\", \"url\": \"https://en.wikipedia.org/wiki/SNAC\", \"lastUpdated\": \" This page was last edited on 2 July 2020, at 13:53\"}\n]" }, { "path": "02_04_b/article_crawler/article_crawler/spiders/articles.xml", "content": "\n\nKevin Baconhttps://en.wikipedia.org/wiki/Kevin_Bacon This page was last edited on 19 September 2020, at 00:35\n (TV series)https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series) This page was last edited on 18 August 2020, at 20:30\nPrimetime Emmy Awardhttps://en.wikipedia.org/wiki/Primetime_Emmy_Award This page was last edited on 22 September 2020, at 10:27\nSixDegrees.orghttps://en.wikipedia.org/wiki/SixDegrees.org This page was last edited on 20 March 2020, at 11:35\nGolden Globe Award for Best Actor – Television Series Musical or Comedyhttps://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy This page was last edited on 14 August 2020, at 04:30\nList of social networking websiteshttps://en.wikipedia.org/wiki/Social_networks This page was last edited on 6 September 2020, at 23:58\nSix Degrees of Kevin Baconhttps://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon This page was last edited on 2 October 2020, at 20:10\nScreen Actors Guild Awardshttps://en.wikipedia.org/wiki/Screen_Actors_Guild_Award This page was last edited on 21 July 2020, at 00:07\n<value><Selector xpath='//h1/i/text()' data='The Guardian'></value>https://en.wikipedia.org/wiki/The_Guardian This page was last edited on 18 September 2020, at 16:08\nHollywood Walk of Famehttps://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame This page was last edited on 3 October 2020, at 12:56\nAcademy Awardshttps://en.wikipedia.org/wiki/Academy_Award This page was last edited on 1 October 2020, at 12:55\nCannes Film Festivalhttps://en.wikipedia.org/wiki/Cannes_Film_Festival This page was last edited on 5 October 2020, at 12:51\n<value><Selector xpath='//h1/i/text()' data='Taking Chance'></value>https://en.wikipedia.org/wiki/Taking_Chance This page was last edited on 3 September 2020, at 14:05\nAlan Rickmanhttps://en.wikipedia.org/wiki/Alan_Rickman This page was last edited on 7 October 2020, at 00:12\n<value><Selector xpath='//h1/i/text()' data='The Following'></value>https://en.wikipedia.org/wiki/The_Following This page was last edited on 11 September 2020, at 16:17\nMain Pagehttps://en.wikipedia.org/wiki/Main_Page This page was last edited on 23 July 2020, at 12:44\nFox Broadcasting Companyhttps://en.wikipedia.org/wiki/Fox_Broadcasting_Company This page was last edited on 6 October 2020, at 15:27\nGolden Globe Awardshttps://en.wikipedia.org/wiki/Golden_Globe_Award This page was last edited on 8 September 2020, at 12:45\nWorldCathttps://en.wikipedia.org/wiki/WorldCat_Identities_(identifier) This page was last edited on 3 October 2020, at 11:46\nVirtual International Authority Filehttps://en.wikipedia.org/wiki/VIAF_(identifier) This page was last edited on 30 July 2020, at 18:19\nHBOhttps://en.wikipedia.org/wiki/HBO This page was last edited on 7 October 2020, at 00:10\nTrovehttps://en.wikipedia.org/wiki/Trove This page was last edited on 18 September 2020, at 03:04\nSystème universitaire de documentationhttps://en.wikipedia.org/wiki/SUDOC_(identifier) This page was last edited on 19 October 2019, at 13:42\n" }, { "path": "02_04_b/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom article_crawler.items import Article\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n custom_settings={\n 'FEED_URI': 'articles.xml',\n 'FEED_FORMAT': 'xml'\n }\n\n def parse_info(self, response):\n article = Article()\n article['title']= response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()')\n article['url'] = response.url\n\n article['lastUpdated'] = response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n return article\n" }, { "path": "02_04_b/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_04_e/article_crawler/article_crawler/__init__.py", "content": "" }, { "path": "02_04_e/article_crawler/article_crawler/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass Article(scrapy.Item):\n title = scrapy.Field()\n url = scrapy.Field()\n lastUpdated = scrapy.Field()\n\n" }, { "path": "02_04_e/article_crawler/article_crawler/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass ArticleCrawlerSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass ArticleCrawlerDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_04_e/article_crawler/article_crawler/pipelines.py", "content": "# -*- coding: utf-8 -*-\n\n# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n\nfrom scrapy.exceptions import DropItem\nfrom datetime import datetime\n\nclass CheckItemPipeline:\n def process_item(self, article, spider):\n if not article['lastUPdated'] or not article['url'] or not article['title']:\n raise DropItem('Missing something!')\n return article\n\n\nclass CleanDatePipeline:\n def process_item(self, article, spider):\n article['lastUpdated'].replace('This page was last edited on', '').strip()\n article['lastUpdated'] = datetime.strptime(article['lastUpdated'], '%d %B %Y, at %H:%M')\n return article\n" }, { "path": "02_04_e/article_crawler/article_crawler/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for article_crawler project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'article_crawler'\n\nCLOSESPIDER_PAGECOUNT=10\n\nFEED_URI='articles.json'\nFEED_FORMAT='json'\n\nSPIDER_MODULES = ['article_crawler.spiders']\nNEWSPIDER_MODULE = 'article_crawler.spiders'\n\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'article_crawler (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = True\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'article_crawler.middlewares.ArticleCrawlerDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\nITEM_PIPELINES = {\n 'article_crawler.pipelines.CheckItemPipeline': 100,\n 'article_crawler.pipelines.CleanDatePipeline': 200,\n\n}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_04_e/article_crawler/article_crawler/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_04_e/article_crawler/article_crawler/spiders/articles.csv", "content": "lastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 26 May 2020, at 18:28\",Obie Award,https://en.wikipedia.org/wiki/Obie_Award\r\n\" This page was last edited on 5 October 2020, at 13:16\",Richard Dean Anderson,https://en.wikipedia.org/wiki/Richard_Dean_Anderson\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 25 September 2020, at 05:22\",\"[]\",https://en.wikipedia.org/wiki/She%27s_Having_a_Baby\r\n\" This page was last edited on 2 July 2020, at 13:53\",SNAC,https://en.wikipedia.org/wiki/SNAC\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 20 September 2020, at 07:23\",[],https://en.wikipedia.org/wiki/Los_Angeles_Daily_News\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 22 July 2020, at 18:39\", (2020 TV series),https://en.wikipedia.org/wiki/Ana_(2020_TV_series)\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n\" This page was last edited on 6 June 2020, at 20:53\",Bruce Gilbert,https://en.wikipedia.org/wiki/Bruce_Gilbert\r\n\" This page was last edited on 23 June 2020, at 19:06\", (TV series),https://en.wikipedia.org/wiki/The_Remix_(TV_series)\r\n\" This page was last edited on 6 October 2020, at 13:04\",[],https://en.wikipedia.org/wiki/The_New_York_Times\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 1 October 2020, at 17:17\", (film),https://en.wikipedia.org/wiki/Black_Mass_(film)\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 11 May 2020, at 14:47\",National Library of Latvia,https://en.wikipedia.org/wiki/National_Library_of_Latvia\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 23 September 2020, at 20:01\", (film),https://en.wikipedia.org/wiki/Patriots_Day_(film)\r\n\" This page was last edited on 5 October 2020, at 15:12\",Judy Garland,https://en.wikipedia.org/wiki/Judy_Garland\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\nlastUpdated,title,url\r\n\" This page was last edited on 19 September 2020, at 00:35\",Kevin Bacon,https://en.wikipedia.org/wiki/Kevin_Bacon\r\n\" This page was last edited on 18 August 2020, at 20:30\", (TV series),https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\r\n\" This page was last edited on 22 September 2020, at 10:27\",Primetime Emmy Award,https://en.wikipedia.org/wiki/Primetime_Emmy_Award\r\n\" This page was last edited on 20 March 2020, at 11:35\",SixDegrees.org,https://en.wikipedia.org/wiki/SixDegrees.org\r\n\" This page was last edited on 2 October 2020, at 20:10\",Six Degrees of Kevin Bacon,https://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon\r\n\" This page was last edited on 1 October 2020, at 12:55\",Academy Awards,https://en.wikipedia.org/wiki/Academy_Award\r\n\" This page was last edited on 18 September 2020, at 16:08\",[],https://en.wikipedia.org/wiki/The_Guardian\r\n\" This page was last edited on 3 October 2020, at 12:56\",Hollywood Walk of Fame,https://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame\r\n\" This page was last edited on 14 August 2020, at 04:30\",Golden Globe Award for Best Actor – Television Series Musical or Comedy,https://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\r\n\" This page was last edited on 3 September 2020, at 14:05\",[],https://en.wikipedia.org/wiki/Taking_Chance\r\n\" This page was last edited on 6 September 2020, at 23:58\",List of social networking websites,https://en.wikipedia.org/wiki/Social_networks\r\n\" This page was last edited on 21 July 2020, at 00:07\",Screen Actors Guild Awards,https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\r\n\" This page was last edited on 8 September 2020, at 12:45\",Golden Globe Awards,https://en.wikipedia.org/wiki/Golden_Globe_Award\r\n\" This page was last edited on 11 September 2020, at 16:17\",[],https://en.wikipedia.org/wiki/The_Following\r\n\" This page was last edited on 6 October 2020, at 03:55\",IMDb,https://en.wikipedia.org/wiki/IMDb\r\n\" This page was last edited on 23 July 2020, at 12:44\",Main Page,https://en.wikipedia.org/wiki/Main_Page\r\n\" This page was last edited on 3 October 2020, at 11:46\",WorldCat,https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\r\n\" This page was last edited on 30 July 2020, at 18:19\",Virtual International Authority File,https://en.wikipedia.org/wiki/VIAF_(identifier)\r\n\" This page was last edited on 18 September 2020, at 03:04\",Trove,https://en.wikipedia.org/wiki/Trove\r\n\" This page was last edited on 6 October 2020, at 15:27\",Fox Broadcasting Company,https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\r\n\" This page was last edited on 7 October 2020, at 00:10\",HBO,https://en.wikipedia.org/wiki/HBO\r\n" }, { "path": "02_04_e/article_crawler/article_crawler/spiders/articles.json", "content": "[\n{\"title\": \"Kevin Bacon\", \"url\": \"https://en.wikipedia.org/wiki/Kevin_Bacon\", \"lastUpdated\": \" This page was last edited on 19 September 2020, at 00:35\"},\n{\"title\": \"Fox Broadcasting Company\", \"url\": \"https://en.wikipedia.org/wiki/Fox_Broadcasting_Company\", \"lastUpdated\": \" This page was last edited on 6 October 2020, at 15:27\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Patriots_Day_(film)\", \"lastUpdated\": \" This page was last edited on 23 September 2020, at 20:01\"},\n{\"title\": \" (TV series)\", \"url\": \"https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)\", \"lastUpdated\": \" This page was last edited on 18 August 2020, at 20:30\"},\n{\"title\": \"Screen Actors Guild Awards\", \"url\": \"https://en.wikipedia.org/wiki/Screen_Actors_Guild_Award\", \"lastUpdated\": \" This page was last edited on 21 July 2020, at 00:07\"},\n,\n,\n{\"title\": \"Primetime Emmy Award\", \"url\": \"https://en.wikipedia.org/wiki/Primetime_Emmy_Award\", \"lastUpdated\": \" This page was last edited on 22 September 2020, at 10:27\"},\n{\"title\": \"Golden Globe Awards\", \"url\": \"https://en.wikipedia.org/wiki/Golden_Globe_Award\", \"lastUpdated\": \" This page was last edited on 8 September 2020, at 12:45\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Black_Mass_(film)\", \"lastUpdated\": \" This page was last edited on 1 October 2020, at 17:17\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Frost/Nixon_(film)\", \"lastUpdated\": \" This page was last edited on 16 August 2020, at 00:08\"},\n,\n,\n{\"title\": \"HBO\", \"url\": \"https://en.wikipedia.org/wiki/HBO\", \"lastUpdated\": \" This page was last edited on 7 October 2020, at 00:10\"},\n,\n{\"title\": \"Circle in the Square Theatre\", \"url\": \"https://en.wikipedia.org/wiki/Circle_in_the_Square\", \"lastUpdated\": \" This page was last edited on 27 September 2020, at 21:06\"},\n{\"title\": \"Main Page\", \"url\": \"https://en.wikipedia.org/wiki/Main_Page\", \"lastUpdated\": \" This page was last edited on 23 July 2020, at 12:44\"},\n{\"title\": \"WorldCat\", \"url\": \"https://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)\", \"lastUpdated\": \" This page was last edited on 3 October 2020, at 11:46\"},\n{\"title\": \"Virtual International Authority File\", \"url\": \"https://en.wikipedia.org/wiki/VIAF_(identifier)\", \"lastUpdated\": \" This page was last edited on 30 July 2020, at 18:19\"},\n{\"title\": \"Trove\", \"url\": \"https://en.wikipedia.org/wiki/Trove\", \"lastUpdated\": \" This page was last edited on 18 September 2020, at 03:04\"},\n{\"title\": \" (film)\", \"url\": \"https://en.wikipedia.org/wiki/Wild_Things_(film)\", \"lastUpdated\": \" This page was last edited on 29 September 2020, at 08:35\"},\n{\"title\": \"Syst\\u00e8me universitaire de documentation\", \"url\": \"https://en.wikipedia.org/wiki/SUDOC_(identifier)\", \"lastUpdated\": \" This page was last edited on 19 October 2019, at 13:42\"},\n{\"title\": \"SNAC\", \"url\": \"https://en.wikipedia.org/wiki/SNAC\", \"lastUpdated\": \" This page was last edited on 2 July 2020, at 13:53\"}\n]" }, { "path": "02_04_e/article_crawler/article_crawler/spiders/articles.xml", "content": "\n\nKevin Baconhttps://en.wikipedia.org/wiki/Kevin_Bacon This page was last edited on 19 September 2020, at 00:35\n (TV series)https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series) This page was last edited on 18 August 2020, at 20:30\nPrimetime Emmy Awardhttps://en.wikipedia.org/wiki/Primetime_Emmy_Award This page was last edited on 22 September 2020, at 10:27\nSixDegrees.orghttps://en.wikipedia.org/wiki/SixDegrees.org This page was last edited on 20 March 2020, at 11:35\nGolden Globe Award for Best Actor – Television Series Musical or Comedyhttps://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy This page was last edited on 14 August 2020, at 04:30\nList of social networking websiteshttps://en.wikipedia.org/wiki/Social_networks This page was last edited on 6 September 2020, at 23:58\nSix Degrees of Kevin Baconhttps://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon This page was last edited on 2 October 2020, at 20:10\nScreen Actors Guild Awardshttps://en.wikipedia.org/wiki/Screen_Actors_Guild_Award This page was last edited on 21 July 2020, at 00:07\n<value><Selector xpath='//h1/i/text()' data='The Guardian'></value>https://en.wikipedia.org/wiki/The_Guardian This page was last edited on 18 September 2020, at 16:08\nHollywood Walk of Famehttps://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame This page was last edited on 3 October 2020, at 12:56\nAcademy Awardshttps://en.wikipedia.org/wiki/Academy_Award This page was last edited on 1 October 2020, at 12:55\nCannes Film Festivalhttps://en.wikipedia.org/wiki/Cannes_Film_Festival This page was last edited on 5 October 2020, at 12:51\n<value><Selector xpath='//h1/i/text()' data='Taking Chance'></value>https://en.wikipedia.org/wiki/Taking_Chance This page was last edited on 3 September 2020, at 14:05\nAlan Rickmanhttps://en.wikipedia.org/wiki/Alan_Rickman This page was last edited on 7 October 2020, at 00:12\n<value><Selector xpath='//h1/i/text()' data='The Following'></value>https://en.wikipedia.org/wiki/The_Following This page was last edited on 11 September 2020, at 16:17\nMain Pagehttps://en.wikipedia.org/wiki/Main_Page This page was last edited on 23 July 2020, at 12:44\nFox Broadcasting Companyhttps://en.wikipedia.org/wiki/Fox_Broadcasting_Company This page was last edited on 6 October 2020, at 15:27\nGolden Globe Awardshttps://en.wikipedia.org/wiki/Golden_Globe_Award This page was last edited on 8 September 2020, at 12:45\nWorldCathttps://en.wikipedia.org/wiki/WorldCat_Identities_(identifier) This page was last edited on 3 October 2020, at 11:46\nVirtual International Authority Filehttps://en.wikipedia.org/wiki/VIAF_(identifier) This page was last edited on 30 July 2020, at 18:19\nHBOhttps://en.wikipedia.org/wiki/HBO This page was last edited on 7 October 2020, at 00:10\nTrovehttps://en.wikipedia.org/wiki/Trove This page was last edited on 18 September 2020, at 03:04\nSystème universitaire de documentationhttps://en.wikipedia.org/wiki/SUDOC_(identifier) This page was last edited on 19 October 2019, at 13:42\n\n\nKevin Baconhttps://en.wikipedia.org/wiki/Kevin_Bacon2020-09-19 00:35:00\n (TV series)https://en.wikipedia.org/wiki/I_Love_Dick_(TV_series)2020-08-18 20:30:00\nSixDegrees.orghttps://en.wikipedia.org/wiki/SixDegrees.org2020-03-20 11:35:00\nGolden Globe Award for Best Actor – Television Series Musical or Comedyhttps://en.wikipedia.org/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy2020-08-14 04:30:00\nList of social networking websiteshttps://en.wikipedia.org/wiki/Social_networks2020-09-06 23:58:00\nSix Degrees of Kevin Baconhttps://en.wikipedia.org/wiki/Six_Degrees_of_Kevin_Bacon2020-10-02 20:10:00\nPrimetime Emmy Awardhttps://en.wikipedia.org/wiki/Primetime_Emmy_Award2020-09-22 10:27:00\nAcademy Awardshttps://en.wikipedia.org/wiki/Academy_Award2020-10-01 12:55:00\nHollywood Walk of Famehttps://en.wikipedia.org/wiki/Hollywood_Walk_of_Fame2020-10-03 12:56:00\n<value><Selector xpath='//h1/i/text()' data='The Guardian'></value>https://en.wikipedia.org/wiki/The_Guardian2020-09-18 16:08:00\n<value><Selector xpath='//h1/i/text()' data='The Following'></value>https://en.wikipedia.org/wiki/The_Following2020-09-11 16:17:00\n<value><Selector xpath='//h1/i/text()' data='Taking Chance'></value>https://en.wikipedia.org/wiki/Taking_Chance2020-09-03 14:05:00\nScreen Actors Guild Awardshttps://en.wikipedia.org/wiki/Screen_Actors_Guild_Award2020-07-21 00:07:00\nSeattle International Film Festivalhttps://en.wikipedia.org/wiki/Seattle_International_Film_Festival2020-04-26 00:39:00\nGolden Globe Awardshttps://en.wikipedia.org/wiki/Golden_Globe_Award2020-09-08 12:45:00\nMain Pagehttps://en.wikipedia.org/wiki/Main_Page2020-07-23 12:44:00\nWorldCathttps://en.wikipedia.org/wiki/WorldCat_Identities_(identifier)2020-10-03 11:46:00\nFox Broadcasting Companyhttps://en.wikipedia.org/wiki/Fox_Broadcasting_Company2020-10-06 15:27:00\nVirtual International Authority Filehttps://en.wikipedia.org/wiki/VIAF_(identifier)2020-07-30 18:19:00\nTrovehttps://en.wikipedia.org/wiki/Trove2020-09-18 03:04:00\nSystème universitaire de documentationhttps://en.wikipedia.org/wiki/SUDOC_(identifier)2019-10-19 13:42:00\nSNAChttps://en.wikipedia.org/wiki/SNAC2020-07-02 13:53:00\nHBOhttps://en.wikipedia.org/wiki/HBO2020-10-07 00:10:00\n" }, { "path": "02_04_e/article_crawler/article_crawler/spiders/wikipedia.py", "content": "# -*- coding: utf-8 -*-\nimport scrapy\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom article_crawler.items import Article\n\nclass WikipediaSpider(CrawlSpider):\n name = 'wikipedia'\n allowed_domains = ['en.wikipedia.org']\n start_urls = ['https://en.wikipedia.org/wiki/Kevin_Bacon']\n\n rules = [\n Rule(LinkExtractor(allow=r'wiki/((?!:).)*$'), callback='parse_info', follow=True)\n ]\n\n custom_settings={\n 'FEED_URI': 'articles.xml',\n 'FEED_FORMAT': 'xml'\n }\n\n def parse_info(self, response):\n article = Article()\n article['title']= response.xpath('//h1/text()').get() or response.xpath('//h1/i/text()')\n article['url'] = response.url\n\n article['lastUpdated'] = response.xpath('//li[@id=\"footer-info-lastmod\"]/text()').get()\n return article\n" }, { "path": "02_04_e/article_crawler/scrapy.cfg", "content": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = article_crawler.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = article_crawler\n" }, { "path": "02_05/news_scraper/news_scraper/__init__.py", "content": "" }, { "path": "02_05/news_scraper/news_scraper/items.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/items.html\n\nimport scrapy\n\n\nclass NewsArticle(scrapy.Item):\n url = scrapy.Field()\n source = scrapy.Field()\n title = scrapy.Field()\n description = scrapy.Field()\n date = scrapy.Field()\n author = scrapy.Field()\n text = scrapy.Field()\n" }, { "path": "02_05/news_scraper/news_scraper/middlewares.py", "content": "# -*- coding: utf-8 -*-\n\n# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nfrom scrapy import signals\n\n\nclass NewsScraperSpiderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the spider middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_spider_input(self, response, spider):\n # Called for each response that goes through the spider\n # middleware and into the spider.\n\n # Should return None or raise an exception.\n return None\n\n def process_spider_output(self, response, result, spider):\n # Called with the results returned from the Spider, after\n # it has processed the response.\n\n # Must return an iterable of Request, dict or Item objects.\n for i in result:\n yield i\n\n def process_spider_exception(self, response, exception, spider):\n # Called when a spider or process_spider_input() method\n # (from other spider middleware) raises an exception.\n\n # Should return either None or an iterable of Request, dict\n # or Item objects.\n pass\n\n def process_start_requests(self, start_requests, spider):\n # Called with the start requests of the spider, and works\n # similarly to the process_spider_output() method, except\n # that it doesn’t have a response associated.\n\n # Must return only requests (not items).\n for r in start_requests:\n yield r\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n\n\nclass NewsScraperDownloaderMiddleware:\n # Not all methods need to be defined. If a method is not defined,\n # scrapy acts as if the downloader middleware does not modify the\n # passed objects.\n\n @classmethod\n def from_crawler(cls, crawler):\n # This method is used by Scrapy to create your spiders.\n s = cls()\n crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)\n return s\n\n def process_request(self, request, spider):\n # Called for each request that goes through the downloader\n # middleware.\n\n # Must either:\n # - return None: continue processing this request\n # - or return a Response object\n # - or return a Request object\n # - or raise IgnoreRequest: process_exception() methods of\n # installed downloader middleware will be called\n return None\n\n def process_response(self, request, response, spider):\n # Called with the response returned from the downloader.\n\n # Must either;\n # - return a Response object\n # - return a Request object\n # - or raise IgnoreRequest\n return response\n\n def process_exception(self, request, exception, spider):\n # Called when a download handler or a process_request()\n # (from other downloader middleware) raises an exception.\n\n # Must either:\n # - return None: continue processing this exception\n # - return a Response object: stops process_exception() chain\n # - return a Request object: stops process_exception() chain\n pass\n\n def spider_opened(self, spider):\n spider.logger.info('Spider opened: %s' % spider.name)\n" }, { "path": "02_05/news_scraper/news_scraper/pipelines.py", "content": "# -*- coding: utf-8 -*-\nfrom datetime import datetime\n\nclass NewsScraperPipeline:\n def process_item(self, item, spider):\n item.date = datetime.strptime(item.date.split('T')[0], '%Y-%B-%D')\n item.author = item.author.replace(', CNN', '')\n item.text = [text.strip() for text in item.text]\n return item\n" }, { "path": "02_05/news_scraper/news_scraper/settings.py", "content": "# -*- coding: utf-8 -*-\n\n# Scrapy settings for news_scraper project\n#\n# For simplicity, this file contains only settings considered important or\n# commonly used. You can find more settings consulting the documentation:\n#\n# https://docs.scrapy.org/en/latest/topics/settings.html\n# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n# https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n\nBOT_NAME = 'news_scraper'\n\nSPIDER_MODULES = ['news_scraper.spiders']\nNEWSPIDER_MODULE = 'news_scraper.spiders'\n\nCLOSESPIDER_PAGECOUNT=10\n\nFEED_URI='news_articles.json'\nFEED_FORMAT='json'\n\n# Crawl responsibly by identifying yourself (and your website) on the user-agent\n#USER_AGENT = 'news_scraper (+http://www.yourdomain.com)'\n\n# Obey robots.txt rules\nROBOTSTXT_OBEY = False\n\n# Configure maximum concurrent requests performed by Scrapy (default: 16)\n#CONCURRENT_REQUESTS = 32\n\n# Configure a delay for requests for the same website (default: 0)\n# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay\n# See also autothrottle settings and docs\n#DOWNLOAD_DELAY = 3\n# The download delay setting will honor only one of:\n#CONCURRENT_REQUESTS_PER_DOMAIN = 16\n#CONCURRENT_REQUESTS_PER_IP = 16\n\n# Disable cookies (enabled by default)\n#COOKIES_ENABLED = False\n\n# Disable Telnet Console (enabled by default)\n#TELNETCONSOLE_ENABLED = False\n\n# Override the default request headers:\n#DEFAULT_REQUEST_HEADERS = {\n# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n# 'Accept-Language': 'en',\n#}\n\n# Enable or disable spider middlewares\n# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html\n#SPIDER_MIDDLEWARES = {\n# 'news_scraper.middlewares.NewsScraperSpiderMiddleware': 543,\n#}\n\n# Enable or disable downloader middlewares\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html\n#DOWNLOADER_MIDDLEWARES = {\n# 'news_scraper.middlewares.NewsScraperDownloaderMiddleware': 543,\n#}\n\n# Enable or disable extensions\n# See https://docs.scrapy.org/en/latest/topics/extensions.html\n#EXTENSIONS = {\n# 'scrapy.extensions.telnet.TelnetConsole': None,\n#}\n\n# Configure item pipelines\n# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html\n#ITEM_PIPELINES = {\n# 'news_scraper.pipelines.NewsScraperPipeline': 300,\n#}\n\n# Enable and configure the AutoThrottle extension (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/autothrottle.html\n#AUTOTHROTTLE_ENABLED = True\n# The initial download delay\n#AUTOTHROTTLE_START_DELAY = 5\n# The maximum download delay to be set in case of high latencies\n#AUTOTHROTTLE_MAX_DELAY = 60\n# The average number of requests Scrapy should be sending in parallel to\n# each remote server\n#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n# Enable showing throttling stats for every response received:\n#AUTOTHROTTLE_DEBUG = False\n\n# Enable and configure HTTP caching (disabled by default)\n# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings\n#HTTPCACHE_ENABLED = True\n#HTTPCACHE_EXPIRATION_SECS = 0\n#HTTPCACHE_DIR = 'httpcache'\n#HTTPCACHE_IGNORE_HTTP_CODES = []\n#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\n" }, { "path": "02_05/news_scraper/news_scraper/spiders/__init__.py", "content": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on how to create and manage\n# your spiders.\n" }, { "path": "02_05/news_scraper/news_scraper/spiders/associated_press.py", "content": "# -*- coding: utf-8 -*-\nfrom scrapy.spiders import CrawlSpider, Rule\nfrom scrapy.linkextractors import LinkExtractor\nfrom news_scraper.items import NewsArticle\nimport json \n\nclass AssociatedPressSpider(CrawlSpider):\n name = 'associated_press'\n allowed_domains = ['apnews.com']\n start_urls = ['http://apnews.com/']\n rules = [Rule(LinkExtractor(allow=r'\\/article\\/[a-zA-Z\\-]+\\-[a-zA-Z0-9]{32}'), callback='parse_item', follow=True)]\n\n def parse_item(self, response):\n article = NewsArticle()\n #