[ { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\npack.sh\n" }, { "path": "DriveDownloader/__init__.py", "content": "" }, { "path": "DriveDownloader/downloader.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nfrom DriveDownloader.netdrives import get_session\r\nfrom DriveDownloader.utils import judge_session, MultiThreadDownloader, judge_scheme\r\nimport argparse\r\nimport os\r\nimport sys\r\nfrom rich.console import Console\r\nfrom rich.progress import (\r\n BarColumn,\r\n DownloadColumn,\r\n Progress,\r\n TaskID,\r\n TextColumn,\r\n TimeRemainingColumn,\r\n TransferSpeedColumn,\r\n)\r\n\r\nMAJOR_VERSION = 1\r\nMINOR_VERSION = 6\r\nPOST_VERSION = 0\r\n__version__ = f\"{MAJOR_VERSION}.{MINOR_VERSION}.{POST_VERSION}\"\r\nconsole = Console(width=72)\r\nurl_scheme_env_key_map = {\r\n \"http\": \"http_proxy\",\r\n \"https\": \"https_proxy\",\r\n}\r\n\r\ndef parse_args():\r\n parser = argparse.ArgumentParser(description='Drive Downloader Args')\r\n parser.add_argument('url', help='URL you want to download from.', default='', type=str)\r\n parser.add_argument('--filename', '-o', help='Target file name.', default='', type=str)\r\n parser.add_argument('--thread-number', '-n', help='thread number of multithread.', type=int, default=1)\r\n parser.add_argument('--version', '-v', action='version', version=__version__, help='Version.')\r\n parser.add_argument('--force-back-google','-F',help='Force to use the backup downloader for GoogleDrive.', action='store_true')\r\n args = parser.parse_args()\r\n return args\r\n\r\ndef get_env(key):\r\n value = os.environ.get(key)\r\n if not value or len(value) == 0:\r\n return None\r\n return value;\r\n\r\ndef download_single_file(url, filename=\"\", thread_number=1, force_back_google=False, list_suffix=None):\r\n scheme = judge_scheme(url)\r\n if scheme not in url_scheme_env_key_map.keys():\r\n raise NotImplementedError(f\"Unsupported scheme {scheme}\")\r\n env_key = url_scheme_env_key_map[scheme]\r\n used_proxy = get_env(env_key)\r\n\r\n session_name = judge_session(url)\r\n session_func = get_session(session_name)\r\n google_fix_logic = False\r\n if session_name == 'GoogleDrive' and thread_number > 1 and not force_back_google:\r\n thread_number = 1\r\n google_fix_logic = True\r\n single_progress = Progress(\r\n TextColumn(\"[bold blue]Downloading: \", justify=\"left\"),\r\n BarColumn(bar_width=15),\r\n \"[progress.percentage]{task.percentage:>3.1f}%\",\r\n \"|\",\r\n DownloadColumn(),\r\n \"|\",\r\n TransferSpeedColumn(),\r\n \"|\",\r\n TimeRemainingColumn(),\r\n refresh_per_second=10\r\n )\r\n multi_progress = Progress(\r\n TextColumn(\"[bold blue]Thread {task.fields[proc_id]}: \", justify=\"left\"),\r\n BarColumn(bar_width=15),\r\n \"[progress.percentage]{task.percentage:>3.1f}%\",\r\n \"|\",\r\n DownloadColumn(),\r\n \"|\",\r\n TransferSpeedColumn(),\r\n \"|\",\r\n TimeRemainingColumn(),\r\n refresh_per_second=10\r\n )\r\n progress_applied = multi_progress if thread_number > 1 else single_progress\r\n download_session = session_func(used_proxy)\r\n download_session.connect(url, filename, force_backup=force_back_google if session_name == 'GoogleDrive' else False)\r\n final_filename = download_session.filename\r\n download_session.show_info(progress_applied, list_suffix)\r\n if google_fix_logic:\r\n console.print('[yellow]Warning: Google Drive URL detected. Only one thread will be created.')\r\n\r\n if thread_number > 1:\r\n download_session = MultiThreadDownloader(progress_applied, session_func, used_proxy, download_session.filesize, thread_number)\r\n interrupted = download_session.get(url, final_filename, force_back_google)\r\n if interrupted:\r\n return\r\n download_session.concatenate(final_filename)\r\n else:\r\n with progress_applied:\r\n task_id = progress_applied.add_task(\"download\", filename=final_filename, proc_id=0, start=False)\r\n interrupted = download_session.save_response_content(progress_bar=progress_applied)\r\n if interrupted:\r\n return\r\n console.print('[green]Bye.')\r\n\r\ndef download_filelist(args):\r\n lines = [line for line in open(args.url, 'r')]\r\n for line_idx, line in enumerate(lines):\r\n splitted_line = line.strip().split(\" \")\r\n url, filename = splitted_line[0], splitted_line[1] if len(splitted_line) > 1 else \"\"\r\n thread_number = int(splitted_line[2]) if len(splitted_line) > 2 else 1\r\n list_suffix = \"({:d}/{:d})\".format(line_idx+1, len(lines))\r\n download_single_file(url, filename, thread_number, args.force_back_google, list_suffix)\r\n\r\ndef simple_cli():\r\n console.print(f\"***********************************************************************\")\r\n console.print(f\"* *\")\r\n console.print(f\"* DriveDownloader {MAJOR_VERSION}.{MINOR_VERSION}.{POST_VERSION} *\")\r\n console.print(f\"* Homesite: https://github.com/hwfan/DriveDownloader *\")\r\n console.print(f\"* *\")\r\n console.print(f\"***********************************************************************\")\r\n args = parse_args()\r\n assert len(args.url) > 0, \"Please input your URL or filelist path!\"\r\n if os.path.exists(args.url):\r\n console.print('Downloading filelist: {:s}'.format(os.path.basename(args.url)))\r\n download_filelist(args)\r\n else:\r\n download_single_file(args.url, args.filename, args.thread_number, args.force_back_google)\r\n\r\nif __name__ == '__main__':\r\n simple_cli()\r\n" }, { "path": "DriveDownloader/netdrives/__init__.py", "content": "from .build import get_session" }, { "path": "DriveDownloader/netdrives/basedrive.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nimport requests\r\nimport requests_random_user_agent\r\nimport sys\r\nimport re\r\nimport os\r\nfrom tqdm import tqdm\r\nfrom DriveDownloader.utils.misc import *\r\nfrom threading import Event\r\nimport signal\r\nfrom rich.console import Console\r\nfrom googleapiclient.http import _retry_request, DEFAULT_CHUNK_SIZE\r\nimport time\r\nimport random\r\n\r\nconsole = Console(width=71)\r\ndone_event = Event()\r\ndef handle_sigint(signum, frame):\r\n console.print(\"\\n[yellow]Interrupted. Will shutdown after the latest chunk is downloaded.\\n\")\r\n done_event.set()\r\nsignal.signal(signal.SIGINT, handle_sigint)\r\n\r\nclass DriveSession:\r\n def __init__(self, proxy=None, chunk_size=32768):\r\n self.session = requests.Session()\r\n self.session.headers['Accept-Encoding'] = ''\r\n if proxy is None:\r\n self.proxies = None\r\n else:\r\n self.proxies = { \"http\": proxy, \"https\": proxy, }\r\n self.params = dict()\r\n self.chunk_size = chunk_size\r\n self.filename = ''\r\n self.filesize = None\r\n self.response = None\r\n self.file_handler = None\r\n self.base_url = None\r\n\r\n def generate_url(self, url):\r\n raise NotImplementedError\r\n \r\n def set_range(self, start, end):\r\n self.session.headers['Range'] = 'bytes={:s}-{:s}'.format(str(start), str(end))\r\n \r\n def parse_response_header(self):\r\n try:\r\n pattern = re.compile(r'filename=\\\"(.*?)\\\"')\r\n filename = pattern.findall(self.response.headers['content-disposition'])[0]\r\n except:\r\n filename = 'noname.out'\r\n\r\n try:\r\n header_size = int(self.response.headers['Content-Length'])\r\n except:\r\n header_size = None\r\n\r\n return filename, header_size\r\n\r\n def save_response_content(self, start=None, end=None, proc_id=-1, progress_bar=None):\r\n dirname = os.path.dirname(self.filename)\r\n if len(dirname) > 0:\r\n os.makedirs(dirname, exist_ok=True)\r\n interrupted = False\r\n\r\n if proc_id >= 0:\r\n name, ext = os.path.splitext(self.filename)\r\n name = name + '_{}'.format(proc_id)\r\n sub_filename = name + ext\r\n sub_dirname = os.path.dirname(sub_filename)\r\n sub_basename = os.path.basename(sub_filename)\r\n sub_tmp_dirname = os.path.join(sub_dirname, 'tmp')\r\n os.makedirs(sub_tmp_dirname, exist_ok=True)\r\n sub_filename = os.path.join(sub_tmp_dirname, sub_basename)\r\n used_filename = sub_filename\r\n else:\r\n proc_id = 0\r\n used_filename = self.filename\r\n start = 0\r\n end = self.filesize-1\r\n\r\n ori_filesize = os.path.getsize(used_filename) if os.path.exists(used_filename) else 0\r\n self.file_handler = open(used_filename, 'ab' if ori_filesize > 0 else 'wb' )\r\n progress_bar.update(proc_id, total=end+1-start)\r\n progress_bar.start_task(proc_id)\r\n progress_bar.update(proc_id, advance=ori_filesize)\r\n\r\n if 'googleapiclient' in str(type(self.response)):\r\n self.chunk_size = 1 * 1024 * 1024\r\n _headers = {}\r\n for k, v in self.response.headers.items():\r\n if not k.lower() in (\"accept\", \"accept-encoding\", \"user-agent\"):\r\n _headers[k] = v\r\n cur_state = start + ori_filesize\r\n while cur_state < end + 1:\r\n headers = _headers.copy()\r\n remained = end + 1 - cur_state\r\n chunk_size = self.chunk_size if remained >= self.chunk_size else remained\r\n headers[\"range\"] = \"bytes=%d-%d\" % (\r\n cur_state,\r\n cur_state + chunk_size - 1,\r\n )\r\n http = self.response.http\r\n resp, content = _retry_request(\r\n http,\r\n 0,\r\n \"media download\",\r\n time.sleep,\r\n random.random,\r\n self.response.uri,\r\n \"GET\",\r\n headers=headers,\r\n )\r\n self.file_handler.write(content)\r\n progress_bar.update(proc_id, advance=len(content))\r\n cur_state += len(content)\r\n if done_event.is_set():\r\n interrupted = True\r\n return interrupted\r\n else:\r\n if ori_filesize > 0:\r\n self.set_range(start + ori_filesize, end)\r\n self.response = self.session.get(self.base_url, params=self.params, proxies=self.proxies, stream=True)\r\n else:\r\n self.set_range(start, end)\r\n cur_state = start + ori_filesize\r\n for chunk in self.response.iter_content(self.chunk_size):\r\n if cur_state >= end + 1:\r\n break\r\n self.file_handler.write(chunk)\r\n chunk_num = len(chunk)\r\n progress_bar.update(proc_id, advance=chunk_num)\r\n cur_state += chunk_num\r\n if done_event.is_set():\r\n interrupted = True\r\n return interrupted\r\n \r\n def connect(self, url, custom_filename=''):\r\n self.base_url = url\r\n self.response = self.session.get(url, params=self.params, proxies=self.proxies, stream=True)\r\n if self.response.status_code // 100 >= 4:\r\n raise RuntimeError(\"Bad status code {}. Please check your connection.\".format(self.response.status_code))\r\n filename_parsed, self.filesize = self.parse_response_header()\r\n self.filename = filename_parsed if len(custom_filename) == 0 else custom_filename\r\n \r\n def show_info(self, progress_bar, list_suffix):\r\n filesize_str = str(format_size(self.filesize)) if self.filesize is not None else 'Invalid'\r\n progress_bar.console.print('{:s}Name: {:s}, Size: {:s}'.format(list_suffix+' ' if list_suffix else '', self.filename, filesize_str))" }, { "path": "DriveDownloader/netdrives/build.py", "content": "#############################################\n# Author: Hongwei Fan #\n# E-mail: hwnorm@outlook.com #\n# Homepage: https://github.com/hwfan #\n#############################################\nfrom .googledrive import GoogleDriveSession\nfrom .onedrive import OneDriveSession\nfrom .sharepoint import SharePointSession\nfrom .dropbox import DropBoxSession\nfrom .directlink import DirectLink\n\n__factory__ = {\"GoogleDrive\": GoogleDriveSession,\n \"OneDrive\": OneDriveSession,\n \"SharePoint\": SharePointSession,\n \"DropBox\": DropBoxSession,\n \"DirectLink\": DirectLink,\n }\n\ndef get_session(name):\n return __factory__[name]\n" }, { "path": "DriveDownloader/netdrives/directlink.py", "content": "#############################################\n# Author: Hongwei Fan #\n# E-mail: hwnorm@outlook.com #\n# Homepage: https://github.com/hwfan #\n#############################################\nimport urllib.parse as urlparse\nimport os\nfrom DriveDownloader.netdrives.basedrive import DriveSession\n\nclass DirectLink(DriveSession):\n def __init__(self, *args, **kwargs):\n DriveSession.__init__(self, *args, **kwargs)\n\n def parse_response_header(self):\n filename = os.path.basename(self.response.url)\n try:\n header_size = int(self.response.headers['Content-Length'])\n except:\n header_size = None\n\n return filename, header_size\n\n def generate_url(self, url):\n return url\n\n def connect(self, url, custom_filename='', proc_id=-1, force_backup=False):\n generated_url = self.generate_url(url)\n DriveSession.connect(self, generated_url, custom_filename=custom_filename)" }, { "path": "DriveDownloader/netdrives/dropbox.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nimport urllib.parse as urlparse\r\nfrom DriveDownloader.netdrives.basedrive import DriveSession\r\n\r\nclass DropBoxSession(DriveSession):\r\n def __init__(self, *args, **kwargs):\r\n DriveSession.__init__(self, *args, **kwargs)\r\n\r\n def generate_url(self, url):\r\n '''\r\n Solution provided by:\r\n https://sunpma.com/564.html\r\n '''\r\n parsed_url = urlparse.urlparse(url)\r\n netloc = parsed_url.netloc.replace('www', 'dl-web')\r\n query = ''\r\n parsed_url = parsed_url._replace(netloc=netloc, query=query)\r\n resultUrl = urlparse.urlunparse(parsed_url)\r\n return resultUrl\r\n\r\n def connect(self, url, custom_filename='', proc_id=-1, force_backup=False):\r\n generated_url = self.generate_url(url)\r\n DriveSession.connect(self, generated_url, custom_filename=custom_filename)" }, { "path": "DriveDownloader/netdrives/googledrive.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nimport urllib.parse as urlparse\r\nfrom DriveDownloader.netdrives.basedrive import DriveSession\r\nfrom DriveDownloader.pydrive2.auth import GoogleAuth\r\nfrom DriveDownloader.pydrive2.drive import GoogleDrive\r\n\r\nimport os\r\nimport sys\r\nfrom rich.console import Console\r\n\r\ngoogleauthdata = \\\r\n'''\r\nclient_config_backend: settings\r\nclient_config:\r\n client_id: 367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com\r\n client_secret: 1qsNodXNaWq1mQuBjUjmvhoO\r\n\r\nsave_credentials: True\r\nsave_credentials_backend: file\r\n\r\nget_refresh_token: True\r\n\r\noauth_scope:\r\n - https://www.googleapis.com/auth/drive\r\n'''\r\n\r\ninfo = \\\r\n'''\r\n+-------------------------------------------------------------------+\r\n|Warning: DriveDownloader is using the backup downloader due to the |\r\n|forbiddance or manual setting. If this is the first time you meet |\r\n|the notice, please follow the instructions to login your Google |\r\n|Account. This operation only needs to be done once. |\r\n+-------------------------------------------------------------------+\r\n'''\r\n\r\nconsole = Console(width=71)\r\nclass GoogleDriveSession(DriveSession):\r\n def __init__(self, *args, **kwargs):\r\n DriveSession.__init__(self, *args, **kwargs)\r\n \r\n def generate_url(self, url):\r\n '''\r\n Solution provided by:\r\n https://stackoverflow.com/questions/25010369/wget-curl-large-file-from-google-drive\r\n '''\r\n parsed_url = urlparse.urlparse(url)\r\n parsed_qs = urlparse.parse_qs(parsed_url.query)\r\n if 'id' in parsed_qs:\r\n id_str = parsed_qs['id'][0]\r\n else:\r\n id_str = parsed_url.path.split('/')[3]\r\n replaced_url = \"https://drive.google.com/u/0/uc?export=download\"\r\n return replaced_url, id_str\r\n\r\n def connect(self, url, custom_filename='', force_backup=False, proc_id=-1):\r\n replaced_url, id_str = self.generate_url(url)\r\n if force_backup:\r\n self.backup_connect(url, custom_filename, id_str, proc_id=proc_id)\r\n return\r\n try:\r\n self.params[\"id\"] = id_str\r\n self.params[\"confirm\"] = \"t\"\r\n DriveSession.connect(self, replaced_url, custom_filename=custom_filename)\r\n except:\r\n self.backup_connect(url, custom_filename, id_str, proc_id=proc_id)\r\n \r\n def backup_connect(self, url, custom_filename, id_str, proc_id=-1):\r\n if proc_id == -1:\r\n console.print(info)\r\n settings_file_path = os.path.join(os.path.dirname(__file__), 'settings.yaml')\r\n if not os.path.exists(settings_file_path):\r\n with open(settings_file_path, \"w\") as f:\r\n f.write(googleauthdata)\r\n self.gauth = GoogleAuth(settings_file=settings_file_path)\r\n self.gauth.CommandLineAuth()\r\n self.gid_str = id_str\r\n drive = GoogleDrive(self.gauth)\r\n file = drive.CreateFile({\"id\": id_str})\r\n self.filename = file['title'] if len(custom_filename) == 0 else custom_filename\r\n self.filesize = float(file['fileSize'])\r\n self.response = self.gauth.service.files().get_media(fileId=id_str)" }, { "path": "DriveDownloader/netdrives/onedrive.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nimport base64\r\nfrom DriveDownloader.netdrives.basedrive import DriveSession\r\n\r\nclass OneDriveSession(DriveSession):\r\n def __init__(self, *args, **kwargs):\r\n DriveSession.__init__(self, *args, **kwargs)\r\n\r\n def generate_url(self, url):\r\n '''\r\n Solution provided by:\r\n https://towardsdatascience.com/how-to-get-onedrive-direct-download-link-ecb52a62fee4\r\n '''\r\n data_bytes64 = base64.b64encode(bytes(url, 'utf-8'))\r\n data_bytes64_String = data_bytes64.decode('utf-8').replace('/','_').replace('+','-').rstrip(\"=\")\r\n resultUrl = f\"https://api.onedrive.com/v1.0/shares/u!{data_bytes64_String}/root/content\"\r\n return resultUrl\r\n\r\n def connect(self, url, custom_filename='', proc_id=-1, force_backup=False):\r\n generated_url = self.generate_url(url)\r\n DriveSession.connect(self, generated_url, custom_filename=custom_filename)" }, { "path": "DriveDownloader/netdrives/settings.yaml", "content": "\nclient_config_backend: settings\nclient_config:\n client_id: 367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com\n client_secret: 1qsNodXNaWq1mQuBjUjmvhoO\n\nsave_credentials: True\nsave_credentials_backend: file\n\nget_refresh_token: True\n\noauth_scope:\n - https://www.googleapis.com/auth/drive\n" }, { "path": "DriveDownloader/netdrives/sharepoint.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nimport urllib.parse as urlparse\r\nfrom DriveDownloader.netdrives.basedrive import DriveSession\r\n\r\nclass SharePointSession(DriveSession):\r\n def __init__(self, *args, **kwargs):\r\n DriveSession.__init__(self, *args, **kwargs)\r\n\r\n def generate_url(self, url):\r\n '''\r\n Solution provided by:\r\n https://www.qian.blue/archives/OneDrive-straight.html\r\n '''\r\n parsed_url = urlparse.urlparse(url)\r\n path = parsed_url.path\r\n netloc = parsed_url.netloc\r\n splitted_path = path.split('/')\r\n personal_attr, domain, sharelink = splitted_path[3:6]\r\n resultUrl = f\"https://{netloc}/{personal_attr}/{domain}/_layouts/52/download.aspx?share={sharelink}\"\r\n return resultUrl\r\n\r\n def connect(self, url, custom_filename='', proc_id=-1, force_backup=False):\r\n generated_url = self.generate_url(url)\r\n DriveSession.connect(self, generated_url, custom_filename=custom_filename)" }, { "path": "DriveDownloader/pydrive2/__init__.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n" }, { "path": "DriveDownloader/pydrive2/apiattr.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nfrom six import Iterator, iteritems\n\n\nclass ApiAttribute(object):\n \"\"\"A data descriptor that sets and returns values.\"\"\"\n\n def __init__(self, name):\n \"\"\"Create an instance of ApiAttribute.\n\n :param name: name of this attribute.\n :type name: str.\n \"\"\"\n self.name = name\n\n def __get__(self, obj, type=None):\n \"\"\"Accesses value of this attribute.\"\"\"\n return obj.attr.get(self.name)\n\n def __set__(self, obj, value):\n \"\"\"Write value of this attribute.\"\"\"\n obj.attr[self.name] = value\n if obj.dirty.get(self.name) is not None:\n obj.dirty[self.name] = True\n\n def __del__(self, obj=None):\n \"\"\"Delete value of this attribute.\"\"\"\n if not obj:\n return\n\n del obj.attr[self.name]\n if obj.dirty.get(self.name) is not None:\n del obj.dirty[self.name]\n\n\nclass ApiAttributeMixin(object):\n \"\"\"Mixin to initialize required global variables to use ApiAttribute.\"\"\"\n\n def __init__(self):\n self.attr = {}\n self.dirty = {}\n self.http = None # Any element may make requests and will require this\n # field.\n\n\nclass ApiResource(dict):\n \"\"\"Super class of all api resources.\n\n Inherits and behaves as a python dictionary to handle api resources.\n Save clean copy of metadata in self.metadata as a dictionary.\n Provides changed metadata elements to efficiently update api resources.\n \"\"\"\n\n auth = ApiAttribute(\"auth\")\n\n def __init__(self, *args, **kwargs):\n \"\"\"Create an instance of ApiResource.\"\"\"\n super(ApiResource, self).__init__()\n self.update(*args, **kwargs)\n self.metadata = dict(self)\n\n def __getitem__(self, key):\n \"\"\"Overwritten method of dictionary.\n\n :param key: key of the query.\n :type key: str.\n :returns: value of the query.\n \"\"\"\n return dict.__getitem__(self, key)\n\n def __setitem__(self, key, val):\n \"\"\"Overwritten method of dictionary.\n\n :param key: key of the query.\n :type key: str.\n :param val: value of the query.\n \"\"\"\n dict.__setitem__(self, key, val)\n\n def __repr__(self):\n \"\"\"Overwritten method of dictionary.\"\"\"\n dict_representation = dict.__repr__(self)\n return \"%s(%s)\" % (type(self).__name__, dict_representation)\n\n def update(self, *args, **kwargs):\n \"\"\"Overwritten method of dictionary.\"\"\"\n for k, v in iteritems(dict(*args, **kwargs)):\n self[k] = v\n\n def UpdateMetadata(self, metadata=None):\n \"\"\"Update metadata and mark all of them to be clean.\"\"\"\n if metadata:\n self.update(metadata)\n self.metadata = dict(self)\n\n def GetChanges(self):\n \"\"\"Returns changed metadata elements to update api resources efficiently.\n\n :returns: dict -- changed metadata elements.\n \"\"\"\n dirty = {}\n for key in self:\n if self.metadata.get(key) is None:\n dirty[key] = self[key]\n elif self.metadata[key] != self[key]:\n dirty[key] = self[key]\n return dirty\n\n\nclass ApiResourceList(ApiAttributeMixin, ApiResource, Iterator):\n \"\"\"Abstract class of all api list resources.\n\n Inherits ApiResource and builds iterator to list any API resource.\n \"\"\"\n\n metadata = ApiAttribute(\"metadata\")\n\n def __init__(self, auth=None, metadata=None):\n \"\"\"Create an instance of ApiResourceList.\n\n :param auth: authorized GoogleAuth instance.\n :type auth: GoogleAuth.\n :param metadata: parameter to send to list command.\n :type metadata: dict.\n \"\"\"\n ApiAttributeMixin.__init__(self)\n ApiResource.__init__(self)\n self.auth = auth\n self.UpdateMetadata()\n if metadata:\n self.update(metadata)\n\n def __iter__(self):\n \"\"\"Returns iterator object.\n\n :returns: ApiResourceList -- self\n \"\"\"\n return self\n\n def __next__(self):\n \"\"\"Make API call to list resources and return them.\n\n Auto updates 'pageToken' every time it makes API call and\n raises StopIteration when it reached the end of iteration.\n\n :returns: list -- list of API resources.\n :raises: StopIteration\n \"\"\"\n if \"pageToken\" in self and self[\"pageToken\"] is None:\n raise StopIteration\n result = self._GetList()\n self[\"pageToken\"] = self.metadata.get(\"nextPageToken\")\n return result\n\n def GetList(self):\n \"\"\"Get list of API resources.\n\n If 'maxResults' is not specified, it will automatically iterate through\n every resources available. Otherwise, it will make API call once and\n update 'pageToken'.\n\n :returns: list -- list of API resources.\n \"\"\"\n if self.get(\"maxResults\") is None:\n self[\"maxResults\"] = 1000\n result = []\n for x in self:\n result.extend(x)\n del self[\"maxResults\"]\n return result\n else:\n return next(self)\n\n def _GetList(self):\n \"\"\"Helper function which actually makes API call.\n\n Should be overwritten.\n\n :raises: NotImplementedError\n \"\"\"\n raise NotImplementedError\n\n def Reset(self):\n \"\"\"Resets current iteration\"\"\"\n if \"pageToken\" in self:\n del self[\"pageToken\"]\n" }, { "path": "DriveDownloader/pydrive2/auth.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nimport socket\nimport webbrowser\nimport httplib2\nimport oauth2client.clientsecrets as clientsecrets\nfrom six.moves import input\nimport threading\n\nfrom googleapiclient.discovery import build\nfrom functools import wraps\nfrom oauth2client.service_account import ServiceAccountCredentials\nfrom oauth2client.client import FlowExchangeError\nfrom oauth2client.client import AccessTokenRefreshError\nfrom oauth2client.client import OAuth2WebServerFlow\nfrom oauth2client.client import OOB_CALLBACK_URN\nfrom oauth2client.file import Storage\nfrom oauth2client.tools import ClientRedirectHandler\nfrom oauth2client.tools import ClientRedirectServer\nfrom oauth2client._helpers import scopes_to_string\nfrom .apiattr import ApiAttribute\nfrom .apiattr import ApiAttributeMixin\nfrom .settings import LoadSettingsFile\nfrom .settings import ValidateSettings\nfrom .settings import SettingsError\nfrom .settings import InvalidConfigError\nimport os\n\nclass AuthError(Exception):\n \"\"\"Base error for authentication/authorization errors.\"\"\"\n\n\nclass InvalidCredentialsError(IOError):\n \"\"\"Error trying to read credentials file.\"\"\"\n\n\nclass AuthenticationRejected(AuthError):\n \"\"\"User rejected authentication.\"\"\"\n\n\nclass AuthenticationError(AuthError):\n \"\"\"General authentication error.\"\"\"\n\n\nclass RefreshError(AuthError):\n \"\"\"Access token refresh error.\"\"\"\n\n\ndef LoadAuth(decoratee):\n \"\"\"Decorator to check if the auth is valid and loads auth if not.\"\"\"\n\n @wraps(decoratee)\n def _decorated(self, *args, **kwargs):\n # Initialize auth if needed.\n if self.auth is None:\n self.auth = GoogleAuth()\n # Re-create access token if it expired.\n if self.auth.access_token_expired:\n if getattr(self.auth, \"auth_method\", False) == \"service\":\n self.auth.ServiceAuth()\n else:\n self.auth.LocalWebserverAuth()\n\n # Initialise service if not built yet.\n if self.auth.service is None:\n self.auth.Authorize()\n\n # Ensure that a thread-safe HTTP object is provided.\n if (\n kwargs is not None\n and \"param\" in kwargs\n and kwargs[\"param\"] is not None\n and \"http\" in kwargs[\"param\"]\n and kwargs[\"param\"][\"http\"] is not None\n ):\n self.http = kwargs[\"param\"][\"http\"]\n del kwargs[\"param\"][\"http\"]\n\n else:\n # If HTTP object not specified, create or resuse an HTTP\n # object from the thread local storage.\n if not getattr(self.auth.thread_local, \"http\", None):\n self.auth.thread_local.http = self.auth.Get_Http_Object()\n self.http = self.auth.thread_local.http\n\n return decoratee(self, *args, **kwargs)\n\n return _decorated\n\n\ndef CheckServiceAuth(decoratee):\n \"\"\"Decorator to authorize service account.\"\"\"\n\n @wraps(decoratee)\n def _decorated(self, *args, **kwargs):\n self.auth_method = \"service\"\n dirty = False\n save_credentials = self.settings.get(\"save_credentials\")\n if self.credentials is None and save_credentials:\n self.LoadCredentials()\n if self.credentials is None:\n decoratee(self, *args, **kwargs)\n self.Authorize()\n dirty = True\n elif self.access_token_expired:\n self.Refresh()\n dirty = True\n if dirty and save_credentials:\n self.SaveCredentials()\n\n return _decorated\n\n\ndef CheckAuth(decoratee):\n \"\"\"Decorator to check if it requires OAuth2 flow request.\"\"\"\n\n @wraps(decoratee)\n def _decorated(self, *args, **kwargs):\n dirty = False\n code = None\n save_credentials = self.settings.get(\"save_credentials\")\n if self.credentials is None and save_credentials:\n self.LoadCredentials()\n if self.flow is None:\n self.GetFlow()\n if self.credentials is None:\n code = decoratee(self, *args, **kwargs)\n dirty = True\n else:\n if self.access_token_expired:\n if self.credentials.refresh_token is not None:\n self.Refresh()\n else:\n code = decoratee(self, *args, **kwargs)\n dirty = True\n if code is not None:\n self.Auth(code)\n if dirty and save_credentials:\n self.SaveCredentials()\n\n return _decorated\n\n\nclass GoogleAuth(ApiAttributeMixin, object):\n \"\"\"Wrapper class for oauth2client library in google-api-python-client.\n\n Loads all settings and credentials from one 'settings.yaml' file\n and performs common OAuth2.0 related functionality such as authentication\n and authorization.\n \"\"\"\n\n DEFAULT_SETTINGS = {\n \"client_config_backend\": \"file\",\n \"client_config_file\": \"client_secrets.json\",\n \"save_credentials\": False,\n \"oauth_scope\": [\"https://www.googleapis.com/auth/drive\"],\n }\n CLIENT_CONFIGS_LIST = [\n \"client_id\",\n \"client_secret\",\n \"auth_uri\",\n \"token_uri\",\n \"revoke_uri\",\n \"redirect_uri\",\n ]\n SERVICE_CONFIGS_LIST = [\"client_user_email\"]\n settings = ApiAttribute(\"settings\")\n client_config = ApiAttribute(\"client_config\")\n flow = ApiAttribute(\"flow\")\n credentials = ApiAttribute(\"credentials\")\n http = ApiAttribute(\"http\")\n service = ApiAttribute(\"service\")\n auth_method = ApiAttribute(\"auth_method\")\n\n def __init__(self, settings_file=\"settings.yaml\", http_timeout=None):\n \"\"\"Create an instance of GoogleAuth.\n\n This constructor just sets the path of settings file.\n It does not actually read the file.\n\n :param settings_file: path of settings file. 'settings.yaml' by default.\n :type settings_file: str.\n \"\"\"\n self.http_timeout = http_timeout\n ApiAttributeMixin.__init__(self)\n self.thread_local = threading.local()\n self.client_config = {}\n try:\n self.settings = LoadSettingsFile(settings_file)\n except SettingsError:\n self.settings = self.DEFAULT_SETTINGS\n else:\n if self.settings is None:\n self.settings = self.DEFAULT_SETTINGS\n else:\n ValidateSettings(self.settings)\n\n @property\n def access_token_expired(self):\n \"\"\"Checks if access token doesn't exist or is expired.\n\n :returns: bool -- True if access token doesn't exist or is expired.\n \"\"\"\n if self.credentials is None:\n return True\n return self.credentials.access_token_expired\n\n @CheckAuth\n def LocalWebserverAuth(\n self, host_name=\"localhost\", port_numbers=None, launch_browser=True\n ):\n \"\"\"Authenticate and authorize from user by creating local web server and\n retrieving authentication code.\n\n This function is not for web server application. It creates local web server\n for user from standalone application.\n\n :param host_name: host name of the local web server.\n :type host_name: str.\n :param port_numbers: list of port numbers to be tried to used.\n :type port_numbers: list.\n :param launch_browser: should browser be launched automatically\n :type launch_browser: bool\n :returns: str -- code returned from local web server\n :raises: AuthenticationRejected, AuthenticationError\n \"\"\"\n if port_numbers is None:\n port_numbers = [\n 8080,\n 8090,\n ] # Mutable objects should not be default\n # values, as each call's changes are global.\n success = False\n port_number = 0\n for port in port_numbers:\n port_number = port\n try:\n httpd = ClientRedirectServer(\n (host_name, port), ClientRedirectHandler\n )\n except socket.error:\n pass\n else:\n success = True\n break\n if success:\n oauth_callback = \"http://%s:%s/\" % (host_name, port_number)\n else:\n print(\n \"Failed to start a local web server. Please check your firewall\"\n )\n print(\n \"settings and locally running programs that may be blocking or\"\n )\n print(\"using configured ports. Default ports are 8080 and 8090.\")\n raise AuthenticationError()\n self.flow.redirect_uri = oauth_callback\n authorize_url = self.GetAuthUrl()\n if launch_browser:\n webbrowser.open(authorize_url, new=1, autoraise=True)\n print(\"Your browser has been opened to visit:\")\n else:\n print(\"Open your browser to visit:\")\n print()\n print(\" \" + authorize_url)\n print()\n httpd.handle_request()\n if \"error\" in httpd.query_params:\n print(\"Authentication request was rejected\")\n raise AuthenticationRejected(\"User rejected authentication\")\n if \"code\" in httpd.query_params:\n return httpd.query_params[\"code\"]\n else:\n print(\n 'Failed to find \"code\" in the query parameters of the redirect.'\n )\n print(\"Try command-line authentication\")\n raise AuthenticationError(\"No code found in redirect\")\n\n @CheckAuth\n def CommandLineAuth(self):\n \"\"\"Authenticate and authorize from user by printing authentication url\n retrieving authentication code from command-line.\n\n :returns: str -- code returned from commandline.\n \"\"\"\n self.flow.redirect_uri = OOB_CALLBACK_URN\n authorize_url = self.GetAuthUrl()\n print(\"Go to the following link in your browser:\")\n print()\n print(\" \" + authorize_url)\n print()\n return input(\"Enter verification code: \").strip()\n\n @CheckServiceAuth\n def ServiceAuth(self):\n \"\"\"Authenticate and authorize using P12 private key, client id\n and client email for a Service account.\n :raises: AuthError, InvalidConfigError\n \"\"\"\n if set(self.SERVICE_CONFIGS_LIST) - set(self.client_config):\n self.LoadServiceConfigSettings()\n scopes = scopes_to_string(self.settings[\"oauth_scope\"])\n client_service_json = self.client_config.get(\"client_json_file_path\")\n if client_service_json:\n self.credentials = ServiceAccountCredentials.from_json_keyfile_name(\n filename=client_service_json, scopes=scopes\n )\n else:\n service_email = self.client_config[\"client_service_email\"]\n file_path = self.client_config[\"client_pkcs12_file_path\"]\n self.credentials = ServiceAccountCredentials.from_p12_keyfile(\n service_account_email=service_email,\n filename=file_path,\n scopes=scopes,\n )\n\n user_email = self.client_config.get(\"client_user_email\")\n if user_email:\n self.credentials = self.credentials.create_delegated(\n sub=user_email\n )\n\n def LoadCredentials(self, backend=None):\n \"\"\"Loads credentials or create empty credentials if it doesn't exist.\n\n :param backend: target backend to save credential to.\n :type backend: str.\n :raises: InvalidConfigError\n \"\"\"\n if backend is None:\n backend = self.settings.get(\"save_credentials_backend\")\n if backend is None:\n raise InvalidConfigError(\"Please specify credential backend\")\n if backend == \"file\":\n self.LoadCredentialsFile()\n else:\n raise InvalidConfigError(\"Unknown save_credentials_backend\")\n\n def LoadCredentialsFile(self, credentials_file=None):\n \"\"\"Loads credentials or create empty credentials if it doesn't exist.\n\n Loads credentials file from path in settings if not specified.\n\n :param credentials_file: path of credentials file to read.\n :type credentials_file: str.\n :raises: InvalidConfigError, InvalidCredentialsError\n \"\"\"\n if credentials_file is None:\n credentials_file = self.settings.get(\"save_credentials_file\")\n if credentials_file is None:\n raise InvalidConfigError(\n \"Please specify credentials file to read\"\n )\n try:\n storage = Storage(credentials_file)\n self.credentials = storage.get()\n except IOError:\n raise InvalidCredentialsError(\n \"Credentials file cannot be symbolic link\"\n )\n\n def SaveCredentials(self, backend=None):\n \"\"\"Saves credentials according to specified backend.\n\n If you have any specific credentials backend in mind, don't use this\n function and use the corresponding function you want.\n\n :param backend: backend to save credentials.\n :type backend: str.\n :raises: InvalidConfigError\n \"\"\"\n if backend is None:\n backend = self.settings.get(\"save_credentials_backend\")\n if backend is None:\n raise InvalidConfigError(\"Please specify credential backend\")\n if backend == \"file\":\n self.SaveCredentialsFile()\n else:\n raise InvalidConfigError(\"Unknown save_credentials_backend\")\n\n def SaveCredentialsFile(self, credentials_file=None):\n \"\"\"Saves credentials to the file in JSON format.\n\n :param credentials_file: destination to save file to.\n :type credentials_file: str.\n :raises: InvalidConfigError, InvalidCredentialsError\n \"\"\"\n if self.credentials is None:\n raise InvalidCredentialsError(\"No credentials to save\")\n if credentials_file is None:\n credentials_file = self.settings.get(\"save_credentials_file\")\n if credentials_file is None:\n raise InvalidConfigError(\n \"Please specify credentials file to read\"\n )\n try:\n storage = Storage(credentials_file)\n storage.put(self.credentials)\n self.credentials.set_store(storage)\n except IOError:\n raise InvalidCredentialsError(\n \"Credentials file cannot be symbolic link\"\n )\n\n def LoadClientConfig(self, backend=None):\n \"\"\"Loads client configuration according to specified backend.\n\n If you have any specific backend to load client configuration from in mind,\n don't use this function and use the corresponding function you want.\n\n :param backend: backend to load client configuration from.\n :type backend: str.\n :raises: InvalidConfigError\n \"\"\"\n if backend is None:\n backend = self.settings.get(\"client_config_backend\")\n if backend is None:\n raise InvalidConfigError(\n \"Please specify client config backend\"\n )\n if backend == \"file\":\n self.LoadClientConfigFile()\n elif backend == \"settings\":\n self.LoadClientConfigSettings()\n elif backend == \"service\":\n self.LoadServiceConfigSettings()\n else:\n raise InvalidConfigError(\"Unknown client_config_backend\")\n\n def LoadClientConfigFile(self, client_config_file=None):\n \"\"\"Loads client configuration file downloaded from APIs console.\n\n Loads client config file from path in settings if not specified.\n\n :param client_config_file: path of client config file to read.\n :type client_config_file: str.\n :raises: InvalidConfigError\n \"\"\"\n if client_config_file is None:\n client_config_file = self.settings[\"client_config_file\"]\n try:\n client_type, client_info = clientsecrets.loadfile(\n client_config_file\n )\n except clientsecrets.InvalidClientSecretsError as error:\n raise InvalidConfigError(\"Invalid client secrets file %s\" % error)\n if client_type not in (\n clientsecrets.TYPE_WEB,\n clientsecrets.TYPE_INSTALLED,\n ):\n raise InvalidConfigError(\n \"Unknown client_type of client config file\"\n )\n\n # General settings.\n try:\n config_index = [\n \"client_id\",\n \"client_secret\",\n \"auth_uri\",\n \"token_uri\",\n ]\n for config in config_index:\n self.client_config[config] = client_info[config]\n\n self.client_config[\"revoke_uri\"] = client_info.get(\"revoke_uri\")\n self.client_config[\"redirect_uri\"] = client_info[\"redirect_uris\"][\n 0\n ]\n except KeyError:\n raise InvalidConfigError(\"Insufficient client config in file\")\n\n # Service auth related fields.\n service_auth_config = [\"client_email\"]\n try:\n for config in service_auth_config:\n self.client_config[config] = client_info[config]\n except KeyError:\n pass # The service auth fields are not present, handling code can go here.\n\n def LoadServiceConfigSettings(self):\n \"\"\"Loads client configuration from settings file.\n :raises: InvalidConfigError\n \"\"\"\n for file_format in [\"json\", \"pkcs12\"]:\n config = f\"client_{file_format}_file_path\"\n value = self.settings[\"service_config\"].get(config)\n if value:\n self.client_config[config] = value\n break\n else:\n raise InvalidConfigError(\n \"Either json or pkcs12 file path required \"\n \"for service authentication\"\n )\n\n if file_format == \"pkcs12\":\n self.SERVICE_CONFIGS_LIST.append(\"client_service_email\")\n\n for config in self.SERVICE_CONFIGS_LIST:\n try:\n self.client_config[config] = self.settings[\"service_config\"][\n config\n ]\n except KeyError:\n err = \"Insufficient service config in settings\"\n err += \"\\n\\nMissing: {} key.\".format(config)\n raise InvalidConfigError(err)\n\n def LoadClientConfigSettings(self):\n \"\"\"Loads client configuration from settings file.\n\n :raises: InvalidConfigError\n \"\"\"\n for config in self.CLIENT_CONFIGS_LIST:\n try:\n self.client_config[config] = self.settings[\"client_config\"][\n config\n ]\n except KeyError:\n raise InvalidConfigError(\n \"Insufficient client config in settings\"\n )\n\n def GetFlow(self):\n \"\"\"Gets Flow object from client configuration.\n\n :raises: InvalidConfigError\n \"\"\"\n if not all(\n config in self.client_config for config in self.CLIENT_CONFIGS_LIST\n ):\n self.LoadClientConfig()\n constructor_kwargs = {\n \"redirect_uri\": self.client_config[\"redirect_uri\"],\n \"auth_uri\": self.client_config[\"auth_uri\"],\n \"token_uri\": self.client_config[\"token_uri\"],\n \"access_type\": \"online\",\n }\n if self.client_config[\"revoke_uri\"] is not None:\n constructor_kwargs[\"revoke_uri\"] = self.client_config[\"revoke_uri\"]\n self.flow = OAuth2WebServerFlow(\n self.client_config[\"client_id\"],\n self.client_config[\"client_secret\"],\n scopes_to_string(self.settings[\"oauth_scope\"]),\n **constructor_kwargs,\n )\n if self.settings.get(\"get_refresh_token\"):\n self.flow.params.update(\n {\"access_type\": \"offline\", \"approval_prompt\": \"force\"}\n )\n\n def Refresh(self):\n \"\"\"Refreshes the access_token.\n\n :raises: RefreshError\n \"\"\"\n if self.credentials is None:\n raise RefreshError(\"No credential to refresh.\")\n if (\n self.credentials.refresh_token is None\n and self.auth_method != \"service\"\n ):\n raise RefreshError(\n \"No refresh_token found.\"\n \"Please set access_type of OAuth to offline.\"\n )\n if self.http is None:\n self.http = self._build_http()\n try:\n self.credentials.refresh(self.http)\n except AccessTokenRefreshError as error:\n raise RefreshError(\"Access token refresh failed: %s\" % error)\n\n def GetAuthUrl(self):\n \"\"\"Creates authentication url where user visits to grant access.\n\n :returns: str -- Authentication url.\n \"\"\"\n if self.flow is None:\n self.GetFlow()\n return self.flow.step1_get_authorize_url()\n\n def Auth(self, code):\n \"\"\"Authenticate, authorize, and build service.\n\n :param code: Code for authentication.\n :type code: str.\n :raises: AuthenticationError\n \"\"\"\n self.Authenticate(code)\n self.Authorize()\n\n def Authenticate(self, code):\n \"\"\"Authenticates given authentication code back from user.\n\n :param code: Code for authentication.\n :type code: str.\n :raises: AuthenticationError\n \"\"\"\n if self.flow is None:\n self.GetFlow()\n try:\n self.credentials = self.flow.step2_exchange(code)\n except FlowExchangeError as e:\n raise AuthenticationError(\"OAuth2 code exchange failed: %s\" % e)\n print(\"Authentication successful.\")\n\n def _build_http(self):\n http = httplib2.Http(timeout=self.http_timeout)\n # 308's are used by several Google APIs (Drive, YouTube)\n # for Resumable Uploads rather than Permanent Redirects.\n # This asks httplib2 to exclude 308s from the status codes\n # it treats as redirects\n # See also: https://stackoverflow.com/a/59850170/298182\n try:\n http.redirect_codes = http.redirect_codes - {308}\n except AttributeError:\n # http.redirect_codes does not exist in previous versions\n # of httplib2, so pass\n pass\n return http\n\n def Authorize(self):\n \"\"\"Authorizes and builds service.\n\n :raises: AuthenticationError\n \"\"\"\n if self.access_token_expired:\n raise AuthenticationError(\n \"No valid credentials provided to authorize\"\n )\n\n if self.http is None:\n self.http = self._build_http()\n self.http = self.credentials.authorize(self.http)\n self.service = build(\n \"drive\", \"v2\", http=self.http, cache_discovery=False\n )\n\n def Get_Http_Object(self):\n \"\"\"Create and authorize an httplib2.Http object. Necessary for\n thread-safety.\n :return: The http object to be used in each call.\n :rtype: httplib2.Http\n \"\"\"\n http = self._build_http()\n http = self.credentials.authorize(http)\n return http\n" }, { "path": "DriveDownloader/pydrive2/drive.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nfrom .apiattr import ApiAttributeMixin\nfrom .files import GoogleDriveFile\nfrom .files import GoogleDriveFileList\nfrom .auth import LoadAuth\n\n\nclass GoogleDrive(ApiAttributeMixin, object):\n \"\"\"Main Google Drive class.\"\"\"\n\n def __init__(self, auth=None):\n \"\"\"Create an instance of GoogleDrive.\n\n :param auth: authorized GoogleAuth instance.\n :type auth: pydrive2.auth.GoogleAuth.\n \"\"\"\n ApiAttributeMixin.__init__(self)\n self.auth = auth\n\n def CreateFile(self, metadata=None):\n \"\"\"Create an instance of GoogleDriveFile with auth of this instance.\n\n This method would not upload a file to GoogleDrive.\n\n :param metadata: file resource to initialize GoogleDriveFile with.\n :type metadata: dict.\n :returns: pydrive2.files.GoogleDriveFile -- initialized with auth of this\n instance.\n \"\"\"\n return GoogleDriveFile(auth=self.auth, metadata=metadata)\n\n def ListFile(self, param=None):\n \"\"\"Create an instance of GoogleDriveFileList with auth of this instance.\n\n This method will not fetch from Files.List().\n\n :param param: parameter to be sent to Files.List().\n :type param: dict.\n :returns: pydrive2.files.GoogleDriveFileList -- initialized with auth of\n this instance.\n \"\"\"\n return GoogleDriveFileList(auth=self.auth, param=param)\n\n @LoadAuth\n def GetAbout(self):\n \"\"\"Return information about the Google Drive of the auth instance.\n\n :returns: A dictionary of Google Drive information like user, usage, quota etc.\n \"\"\"\n return self.auth.service.about().get().execute(http=self.http)\n" }, { "path": "DriveDownloader/pydrive2/files.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nimport io\nimport mimetypes\nimport json\n\nfrom googleapiclient import errors\nfrom googleapiclient.http import MediaIoBaseUpload\nfrom googleapiclient.http import MediaIoBaseDownload\nfrom googleapiclient.http import DEFAULT_CHUNK_SIZE\nfrom functools import wraps\n\nfrom .apiattr import ApiAttribute\nfrom .apiattr import ApiAttributeMixin\nfrom .apiattr import ApiResource\nfrom .apiattr import ApiResourceList\nfrom .auth import LoadAuth\n\nBLOCK_SIZE = 1024\n# Usage: MIME_TYPE_TO_BOM[''][''].\nMIME_TYPE_TO_BOM = {\n \"application/vnd.google-apps.document\": {\n \"text/plain\": u\"\\ufeff\".encode(\"utf8\")\n }\n}\n\n\nclass FileNotUploadedError(RuntimeError):\n \"\"\"Error trying to access metadata of file that is not uploaded.\"\"\"\n\n\nclass ApiRequestError(IOError):\n def __init__(self, http_error):\n assert isinstance(http_error, errors.HttpError)\n content = json.loads(http_error.content.decode(\"utf-8\"))\n self.error = content.get(\"error\", {}) if content else {}\n\n # Initialize args for backward compatibility\n super().__init__(http_error)\n\n def GetField(self, field):\n \"\"\"Returns the `field` from the first error\"\"\"\n return self.error.get(\"errors\", [{}])[0].get(field, \"\")\n\n\nclass FileNotDownloadableError(RuntimeError):\n \"\"\"Error trying to download file that is not downloadable.\"\"\"\n\n\ndef LoadMetadata(decoratee):\n \"\"\"Decorator to check if the file has metadata and fetches it if not.\n\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n\n @wraps(decoratee)\n def _decorated(self, *args, **kwargs):\n if not self.uploaded:\n self.FetchMetadata()\n return decoratee(self, *args, **kwargs)\n\n return _decorated\n\n\nclass GoogleDriveFileList(ApiResourceList):\n \"\"\"Google Drive FileList instance.\n\n Equivalent to Files.list() in Drive APIs.\n \"\"\"\n\n def __init__(self, auth=None, param=None):\n \"\"\"Create an instance of GoogleDriveFileList.\"\"\"\n super(GoogleDriveFileList, self).__init__(auth=auth, metadata=param)\n\n @LoadAuth\n def _GetList(self):\n \"\"\"Overwritten method which actually makes API call to list files.\n\n :returns: list -- list of pydrive2.files.GoogleDriveFile.\n \"\"\"\n # Teamdrive support\n self[\"supportsAllDrives\"] = True\n self[\"includeItemsFromAllDrives\"] = True\n\n try:\n self.metadata = (\n self.auth.service.files()\n .list(**dict(self))\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n\n result = []\n for file_metadata in self.metadata[\"items\"]:\n tmp_file = GoogleDriveFile(\n auth=self.auth, metadata=file_metadata, uploaded=True\n )\n result.append(tmp_file)\n return result\n\n\nclass IoBuffer(object):\n \"\"\"Lightweight retention of one chunk.\"\"\"\n\n def __init__(self, encoding):\n self.encoding = encoding\n self.chunk = None\n\n def write(self, chunk):\n self.chunk = chunk\n\n def read(self):\n return (\n self.chunk.decode(self.encoding)\n if self.chunk and self.encoding\n else self.chunk\n )\n\n\nclass MediaIoReadable(object):\n def __init__(\n self,\n request,\n encoding=None,\n pre_buffer=True,\n remove_prefix=b\"\",\n chunksize=DEFAULT_CHUNK_SIZE,\n ):\n \"\"\"File-like wrapper around MediaIoBaseDownload.\n\n :param pre_buffer: Whether to read one chunk into an internal buffer\n immediately in order to raise any potential errors.\n :param remove_prefix: Bytes prefix to remove from internal pre_buffer.\n :raises: ApiRequestError\n \"\"\"\n self.done = False\n self._fd = IoBuffer(encoding)\n self.downloader = MediaIoBaseDownload(\n self._fd, request, chunksize=chunksize\n )\n self.size = None\n self._pre_buffer = False\n if pre_buffer:\n self.read()\n if remove_prefix:\n chunk = io.BytesIO(self._fd.chunk)\n GoogleDriveFile._RemovePrefix(chunk, remove_prefix)\n self._fd.chunk = chunk.getvalue()\n self._pre_buffer = True\n\n def read(self):\n \"\"\"\n :returns: bytes or str -- chunk (or None if done)\n :raises: ApiRequestError\n \"\"\"\n if self._pre_buffer:\n self._pre_buffer = False\n return self._fd.read()\n if self.done:\n return None\n try:\n status, self.done = self.downloader.next_chunk()\n self.size = status.total_size\n except errors.HttpError as error:\n raise ApiRequestError(error)\n return self._fd.read()\n\n def __iter__(self):\n \"\"\"\n :raises: ApiRequestError\n \"\"\"\n while True:\n chunk = self.read()\n if chunk is None:\n break\n yield chunk\n\n def __len__(self):\n return self.size\n\n\nclass GoogleDriveFile(ApiAttributeMixin, ApiResource):\n \"\"\"Google Drive File instance.\n\n Inherits ApiResource which inherits dict.\n Can access and modify metadata like dictionary.\n \"\"\"\n\n content = ApiAttribute(\"content\")\n uploaded = ApiAttribute(\"uploaded\")\n metadata = ApiAttribute(\"metadata\")\n\n def __init__(self, auth=None, metadata=None, uploaded=False):\n \"\"\"Create an instance of GoogleDriveFile.\n\n :param auth: authorized GoogleAuth instance.\n :type auth: pydrive2.auth.GoogleAuth\n :param metadata: file resource to initialize GoogleDriveFile with.\n :type metadata: dict.\n :param uploaded: True if this file is confirmed to be uploaded.\n :type uploaded: bool.\n \"\"\"\n ApiAttributeMixin.__init__(self)\n ApiResource.__init__(self)\n self.metadata = {}\n self.dirty = {\"content\": False}\n self.auth = auth\n self.uploaded = uploaded\n if uploaded:\n self.UpdateMetadata(metadata)\n elif metadata:\n self.update(metadata)\n self.has_bom = True\n\n def __getitem__(self, key):\n \"\"\"Overwrites manner of accessing Files resource.\n\n If this file instance is not uploaded and id is specified,\n it will try to look for metadata with Files.get().\n\n :param key: key of dictionary query.\n :type key: str.\n :returns: value of Files resource\n :raises: KeyError, FileNotUploadedError\n \"\"\"\n try:\n return dict.__getitem__(self, key)\n except KeyError as e:\n if self.uploaded:\n raise KeyError(e)\n if self.get(\"id\"):\n self.FetchMetadata()\n return dict.__getitem__(self, key)\n else:\n raise FileNotUploadedError()\n\n def SetContentString(self, content, encoding=\"utf-8\"):\n \"\"\"Set content of this file to be a string.\n\n Creates io.BytesIO instance of utf-8 encoded string.\n Sets mimeType to be 'text/plain' if not specified.\n\n :param encoding: The encoding to use when setting the content of this file.\n :type encoding: str\n :param content: content of the file in string.\n :type content: str\n \"\"\"\n self.content = io.BytesIO(content.encode(encoding))\n if self.get(\"mimeType\") is None:\n self[\"mimeType\"] = \"text/plain\"\n\n def SetContentFile(self, filename):\n \"\"\"Set content of this file from a file.\n\n Opens the file specified by this method.\n Will be read, uploaded, and closed by Upload() method.\n Sets metadata 'title' and 'mimeType' automatically if not specified.\n\n :param filename: name of the file to be uploaded.\n :type filename: str.\n \"\"\"\n self.content = open(filename, \"rb\")\n if self.get(\"title\") is None:\n self[\"title\"] = filename\n if self.get(\"mimeType\") is None:\n self[\"mimeType\"] = mimetypes.guess_type(filename)[0]\n\n def GetContentString(\n self, mimetype=None, encoding=\"utf-8\", remove_bom=False\n ):\n \"\"\"Get content of this file as a string.\n\n :param mimetype: The mimetype of the content string.\n :type mimetype: str\n\n :param encoding: The encoding to use when decoding the byte string.\n :type encoding: str\n\n :param remove_bom: Whether to strip a known BOM.\n :type remove_bom: bool\n\n :returns: str -- utf-8 decoded content of the file\n :raises: ApiRequestError, FileNotUploadedError, FileNotDownloadableError\n \"\"\"\n if (\n self.content is None\n or type(self.content) is not io.BytesIO\n or self.has_bom == remove_bom\n ):\n self.FetchContent(mimetype, remove_bom)\n return self.content.getvalue().decode(encoding)\n\n @LoadAuth\n def GetContentFile(\n self,\n filename,\n mimetype=None,\n remove_bom=False,\n callback=None,\n chunksize=DEFAULT_CHUNK_SIZE,\n ):\n \"\"\"Save content of this file as a local file.\n\n :param filename: name of the file to write to.\n :type filename: str\n :param mimetype: mimeType of the file.\n :type mimetype: str\n :param remove_bom: Whether to remove the byte order marking.\n :type remove_bom: bool\n :param callback: passed two arguments: (total transferred, file size).\n :type param: callable\n :param chunksize: chunksize in bytes (standard 100 MB(1024*1024*100))\n :type chunksize: int\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n files = self.auth.service.files()\n file_id = self.metadata.get(\"id\") or self.get(\"id\")\n if not file_id:\n raise FileNotUploadedError()\n\n def download(fd, request):\n downloader = MediaIoBaseDownload(\n fd, self._WrapRequest(request), chunksize=chunksize\n )\n done = False\n while done is False:\n status, done = downloader.next_chunk()\n if callback:\n callback(status.resumable_progress, status.total_size)\n\n with open(filename, mode=\"w+b\") as fd:\n # Should use files.export_media instead of files.get_media if\n # metadata[\"mimeType\"].startswith(\"application/vnd.google-apps.\").\n # But that would first require a slow call to FetchMetadata().\n # We prefer to try-except for speed.\n try:\n download(fd, files.get_media(fileId=file_id))\n except errors.HttpError as error:\n exc = ApiRequestError(error)\n if (\n exc.error[\"code\"] != 403\n or exc.GetField(\"reason\") != \"fileNotDownloadable\"\n ):\n raise exc\n mimetype = mimetype or \"text/plain\"\n fd.seek(0) # just in case `download()` modified `fd`\n try:\n download(\n fd,\n files.export_media(fileId=file_id, mimeType=mimetype),\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n\n if mimetype == \"text/plain\" and remove_bom:\n fd.seek(0)\n bom = self._GetBOM(mimetype)\n if bom:\n self._RemovePrefix(fd, bom)\n\n @LoadAuth\n def GetContentIOBuffer(\n self,\n mimetype=None,\n encoding=None,\n remove_bom=False,\n chunksize=DEFAULT_CHUNK_SIZE,\n ):\n \"\"\"Get a file-like object which has a buffered read() method.\n\n :param mimetype: mimeType of the file.\n :type mimetype: str\n :param encoding: The encoding to use when decoding the byte string.\n :type encoding: str\n :param remove_bom: Whether to remove the byte order marking.\n :type remove_bom: bool\n :param chunksize: default read()/iter() chunksize.\n :type chunksize: int\n :returns: MediaIoReadable -- file-like object.\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n files = self.auth.service.files()\n file_id = self.metadata.get(\"id\") or self.get(\"id\")\n if not file_id:\n raise FileNotUploadedError()\n\n # Should use files.export_media instead of files.get_media if\n # metadata[\"mimeType\"].startswith(\"application/vnd.google-apps.\").\n # But that would first require a slow call to FetchMetadata().\n # We prefer to try-except for speed.\n try:\n request = self._WrapRequest(files.get_media(fileId=file_id))\n return MediaIoReadable(\n request, encoding=encoding, chunksize=chunksize\n )\n except ApiRequestError as exc:\n if (\n exc.error[\"code\"] != 403\n or exc.GetField(\"reason\") != \"fileNotDownloadable\"\n ):\n raise exc\n mimetype = mimetype or \"text/plain\"\n request = self._WrapRequest(\n files.export_media(fileId=file_id, mimeType=mimetype)\n )\n remove_prefix = (\n self._GetBOM(mimetype)\n if mimetype == \"text/plain\" and remove_bom\n else b\"\"\n )\n return MediaIoReadable(\n request,\n encoding=encoding,\n remove_prefix=remove_prefix,\n chunksize=chunksize,\n )\n\n @LoadAuth\n def FetchMetadata(self, fields=None, fetch_all=False):\n \"\"\"Download file's metadata from id using Files.get().\n\n :param fields: The fields to include, as one string, each entry separated\n by commas, e.g. 'fields,labels'.\n :type fields: str\n\n :param fetch_all: Whether to fetch all fields.\n :type fetch_all: bool\n\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n file_id = self.metadata.get(\"id\") or self.get(\"id\")\n\n if fetch_all:\n fields = \"*\"\n\n if file_id:\n try:\n metadata = (\n self.auth.service.files()\n .get(\n fileId=file_id,\n fields=fields,\n # Teamdrive support\n supportsAllDrives=True,\n )\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n self.uploaded = True\n self.UpdateMetadata(metadata)\n else:\n raise FileNotUploadedError()\n\n @LoadMetadata\n def FetchContent(self, mimetype=None, remove_bom=False):\n \"\"\"Download file's content from download_url.\n\n :raises: ApiRequestError, FileNotUploadedError, FileNotDownloadableError\n \"\"\"\n download_url = self.metadata.get(\"downloadUrl\")\n export_links = self.metadata.get(\"exportLinks\")\n if download_url:\n self.content = io.BytesIO(self._DownloadFromUrl(download_url))\n self.dirty[\"content\"] = False\n\n elif export_links and export_links.get(mimetype):\n self.content = io.BytesIO(\n self._DownloadFromUrl(export_links.get(mimetype))\n )\n self.dirty[\"content\"] = False\n\n else:\n raise FileNotDownloadableError(\n \"No downloadLink/exportLinks for mimetype found in metadata\"\n )\n\n if mimetype == \"text/plain\" and remove_bom:\n self._RemovePrefix(\n self.content, MIME_TYPE_TO_BOM[self[\"mimeType\"]][mimetype]\n )\n self.has_bom = not remove_bom\n\n def Upload(self, param=None):\n \"\"\"Upload/update file by choosing the most efficient method.\n\n :param param: additional parameter to upload file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n if self.uploaded or self.get(\"id\") is not None:\n if self.dirty[\"content\"]:\n self._FilesUpdate(param=param)\n else:\n self._FilesPatch(param=param)\n else:\n self._FilesInsert(param=param)\n\n def Trash(self, param=None):\n \"\"\"Move a file to the trash.\n\n :raises: ApiRequestError\n \"\"\"\n self._FilesTrash(param=param)\n\n def UnTrash(self, param=None):\n \"\"\"Move a file out of the trash.\n :param param: Additional parameter to file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n self._FilesUnTrash(param=param)\n\n def Delete(self, param=None):\n \"\"\"Hard-delete a file.\n\n :param param: additional parameter to file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n self._FilesDelete(param=param)\n\n def InsertPermission(self, new_permission, param=None):\n \"\"\"Insert a new permission. Re-fetches all permissions after call.\n\n :param new_permission: The new permission to insert, please see the\n official Google Drive API guide on permissions.insert\n for details.\n :type new_permission: object\n\n :param param: addition parameters to pass\n :type param: dict\n\n :return: The permission object.\n :rtype: object\n \"\"\"\n if param is None:\n param = {}\n param[\"fileId\"] = self.metadata.get(\"id\") or self[\"id\"]\n param[\"body\"] = new_permission\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n permission = (\n self.auth.service.permissions()\n .insert(**param)\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n self.GetPermissions() # Update permissions field.\n\n return permission\n\n @LoadAuth\n def GetPermissions(self):\n \"\"\"Get file's or shared drive's permissions.\n\n For files in a shared drive, at most 100 results will be returned.\n It doesn't paginate and collect all results.\n\n :return: A list of the permission objects.\n :rtype: object[]\n \"\"\"\n file_id = self.metadata.get(\"id\") or self.get(\"id\")\n\n # We can't do FetchMetada call (which would nicely update\n # local metada cache, etc) here since it doesn't return\n # permissions for the team drive use case.\n permissions = (\n self.auth.service.permissions()\n .list(\n fileId=file_id,\n # Teamdrive support\n supportsAllDrives=True,\n )\n .execute(http=self.http)\n ).get(\"items\")\n\n if permissions:\n self[\"permissions\"] = permissions\n self.metadata[\"permissions\"] = permissions\n\n return permissions\n\n def DeletePermission(self, permission_id):\n \"\"\"Deletes the permission specified by the permission_id.\n\n :param permission_id: The permission id.\n :type permission_id: str\n :return: True if it succeeds.\n :rtype: bool\n \"\"\"\n return self._DeletePermission(permission_id)\n\n def _WrapRequest(self, request):\n \"\"\"Replaces request.http with self.http.\n\n Ensures thread safety. Similar to other places where we call\n `.execute(http=self.http)` to pass a client from the thread local storage.\n \"\"\"\n if self.http:\n request.http = self.http\n return request\n\n @LoadAuth\n def _FilesInsert(self, param=None):\n \"\"\"Upload a new file using Files.insert().\n\n :param param: additional parameter to upload file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n if param is None:\n param = {}\n param[\"body\"] = self.GetChanges()\n\n # teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n if self.dirty[\"content\"]:\n param[\"media_body\"] = self._BuildMediaBody()\n metadata = (\n self.auth.service.files()\n .insert(**param)\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n self.uploaded = True\n self.dirty[\"content\"] = False\n self.UpdateMetadata(metadata)\n\n @LoadAuth\n def _FilesUnTrash(self, param=None):\n \"\"\"Un-delete (Trash) a file using Files.UnTrash().\n :param param: additional parameter to file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n if param is None:\n param = {}\n param[\"fileId\"] = self.metadata.get(\"id\") or self[\"id\"]\n\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n self.auth.service.files().untrash(**param).execute(http=self.http)\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n if self.metadata:\n self.metadata[u\"labels\"][u\"trashed\"] = False\n return True\n\n @LoadAuth\n def _FilesTrash(self, param=None):\n \"\"\"Soft-delete (Trash) a file using Files.Trash().\n\n :param param: additional parameter to file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n if param is None:\n param = {}\n param[\"fileId\"] = self.metadata.get(\"id\") or self[\"id\"]\n\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n self.auth.service.files().trash(**param).execute(http=self.http)\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n if self.metadata:\n self.metadata[u\"labels\"][u\"trashed\"] = True\n return True\n\n @LoadAuth\n def _FilesDelete(self, param=None):\n \"\"\"Delete a file using Files.Delete()\n (WARNING: deleting permanently deletes the file!)\n\n :param param: additional parameter to file.\n :type param: dict.\n :raises: ApiRequestError\n \"\"\"\n if param is None:\n param = {}\n param[\"fileId\"] = self.metadata.get(\"id\") or self[\"id\"]\n\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n self.auth.service.files().delete(**param).execute(http=self.http)\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n return True\n\n @LoadAuth\n @LoadMetadata\n def _FilesUpdate(self, param=None):\n \"\"\"Update metadata and/or content using Files.Update().\n\n :param param: additional parameter to upload file.\n :type param: dict.\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n if param is None:\n param = {}\n param[\"body\"] = self.GetChanges()\n param[\"fileId\"] = self.metadata.get(\"id\")\n\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n if self.dirty[\"content\"]:\n param[\"media_body\"] = self._BuildMediaBody()\n metadata = (\n self.auth.service.files()\n .update(**param)\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n self.uploaded = True\n self.dirty[\"content\"] = False\n self.UpdateMetadata(metadata)\n\n @LoadAuth\n @LoadMetadata\n def _FilesPatch(self, param=None):\n \"\"\"Update metadata using Files.Patch().\n\n :param param: additional parameter to upload file.\n :type param: dict.\n :raises: ApiRequestError, FileNotUploadedError\n \"\"\"\n if param is None:\n param = {}\n param[\"body\"] = self.GetChanges()\n param[\"fileId\"] = self.metadata.get(\"id\")\n\n # Teamdrive support\n param[\"supportsAllDrives\"] = True\n\n try:\n metadata = (\n self.auth.service.files()\n .patch(**param)\n .execute(http=self.http)\n )\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n self.UpdateMetadata(metadata)\n\n def _BuildMediaBody(self):\n \"\"\"Build MediaIoBaseUpload to get prepared to upload content of the file.\n\n Sets mimeType as 'application/octet-stream' if not specified.\n\n :returns: MediaIoBaseUpload -- instance that will be used to upload content.\n \"\"\"\n if self.get(\"mimeType\") is None:\n self[\"mimeType\"] = \"application/octet-stream\"\n return MediaIoBaseUpload(\n self.content, self[\"mimeType\"], resumable=True\n )\n\n @LoadAuth\n def _DownloadFromUrl(self, url):\n \"\"\"Download file from url using provided credential.\n\n :param url: link of the file to download.\n :type url: str.\n :returns: str -- content of downloaded file in string.\n :raises: ApiRequestError\n \"\"\"\n resp, content = self.http.request(url)\n if resp.status != 200:\n raise ApiRequestError(errors.HttpError(resp, content, uri=url))\n return content\n\n @LoadAuth\n def _DeletePermission(self, permission_id):\n \"\"\"Deletes the permission remotely, and from the file object itself.\n\n :param permission_id: The ID of the permission.\n :type permission_id: str\n\n :return: The permission\n :rtype: object\n \"\"\"\n file_id = self.metadata.get(\"id\") or self[\"id\"]\n try:\n self.auth.service.permissions().delete(\n fileId=file_id, permissionId=permission_id\n ).execute()\n except errors.HttpError as error:\n raise ApiRequestError(error)\n else:\n if \"permissions\" in self and \"permissions\" in self.metadata:\n permissions = self[\"permissions\"]\n is_not_current_permission = (\n lambda per: per[\"id\"] == permission_id\n )\n permissions = list(\n filter(is_not_current_permission, permissions)\n )\n self[\"permissions\"] = permissions\n self.metadata[\"permissions\"] = permissions\n return True\n\n @staticmethod\n def _GetBOM(mimetype):\n \"\"\"Based on download mime type (ignores Google Drive mime type)\"\"\"\n for bom in MIME_TYPE_TO_BOM.values():\n if mimetype in bom:\n return bom[mimetype]\n\n @staticmethod\n def _RemovePrefix(file_object, prefix, block_size=BLOCK_SIZE):\n \"\"\"Deletes passed prefix by shifting content of passed file object by to\n the left. Operation is in-place.\n\n Args:\n file_object (obj): The file object to manipulate.\n prefix (str): The prefix to insert.\n block_size (int): The size of the blocks which are moved one at a time.\n \"\"\"\n prefix_length = len(prefix)\n # Detect if prefix exists in file.\n content_start = file_object.read(prefix_length)\n\n if content_start == prefix:\n # Shift content left by prefix length, by copying 1KiB at a time.\n block_to_write = file_object.read(block_size)\n current_block_length = len(block_to_write)\n\n # Read and write location in separate variables for simplicity.\n read_location = prefix_length + current_block_length\n write_location = 0\n\n while current_block_length > 0:\n # Write next block.\n file_object.seek(write_location)\n file_object.write(block_to_write)\n # Set write location to the next block.\n write_location += len(block_to_write)\n\n # Read next block of input.\n file_object.seek(read_location)\n block_to_write = file_object.read(block_size)\n # Update the current block length and read_location.\n current_block_length = len(block_to_write)\n read_location += current_block_length\n\n # Truncate the file to its, now shorter, length.\n file_object.truncate(read_location - prefix_length)\n\n @staticmethod\n def _InsertPrefix(file_object, prefix, block_size=BLOCK_SIZE):\n \"\"\"Inserts the passed prefix in the beginning of the file, operation is\n in-place.\n\n Args:\n file_object (obj): The file object to manipulate.\n prefix (str): The prefix to insert.\n \"\"\"\n # Read the first two blocks.\n first_block = file_object.read(block_size)\n second_block = file_object.read(block_size)\n # Pointer to the first byte of the next block to be read.\n read_location = block_size * 2\n\n # Write BOM.\n file_object.seek(0)\n file_object.write(prefix)\n # {read|write}_location separated for readability.\n write_location = len(prefix)\n\n # Write and read block alternatingly.\n while len(first_block):\n # Write first block.\n file_object.seek(write_location)\n file_object.write(first_block)\n # Increment write_location.\n write_location += block_size\n\n # Move second block into first variable.\n first_block = second_block\n\n # Read in the next block.\n file_object.seek(read_location)\n second_block = file_object.read(block_size)\n # Increment read_location.\n read_location += block_size\n" }, { "path": "DriveDownloader/pydrive2/fs/__init__.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nfrom pydrive2.fs.spec import GDriveFileSystem\n\n__all__ = [\"GDriveFileSystem\"]\n" }, { "path": "DriveDownloader/pydrive2/fs/spec.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nimport errno\nimport io\nimport logging\nimport os\nimport posixpath\nimport threading\nfrom collections import defaultdict\n\nfrom fsspec.spec import AbstractFileSystem\nfrom funcy import cached_property, retry, wrap_prop, wrap_with\nfrom funcy.py3 import cat\nfrom tqdm.utils import CallbackIOWrapper\n\nfrom pydrive2.drive import GoogleDrive\nfrom pydrive2.fs.utils import IterStream\n\nlogger = logging.getLogger(__name__)\n\nFOLDER_MIME_TYPE = \"application/vnd.google-apps.folder\"\n\n\ndef _gdrive_retry(func):\n def should_retry(exc):\n from pydrive2.files import ApiRequestError\n\n if not isinstance(exc, ApiRequestError):\n return False\n\n error_code = exc.error.get(\"code\", 0)\n result = False\n if 500 <= error_code < 600:\n result = True\n\n if error_code == 403:\n result = exc.GetField(\"reason\") in [\n \"userRateLimitExceeded\",\n \"rateLimitExceeded\",\n ]\n if result:\n logger.debug(f\"Retrying GDrive API call, error: {exc}.\")\n\n return result\n\n # 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s\n return retry(\n 16,\n timeout=lambda a: min(0.5 * 1.618 ** a, 20),\n filter_errors=should_retry,\n )(func)\n\n\nclass GDriveFileSystem(AbstractFileSystem):\n def __init__(self, path, google_auth, trash_only=True, **kwargs):\n self.path = path\n self.root, self.base = self.split_path(self.path)\n self.client = GoogleDrive(google_auth)\n self._trash_only = trash_only\n super().__init__(**kwargs)\n\n def split_path(self, path):\n parts = path.replace(\"//\", \"/\").rstrip(\"/\").split(\"/\", 1)\n if len(parts) == 2:\n return parts\n else:\n return parts[0], \"\"\n\n @wrap_prop(threading.RLock())\n @cached_property\n def _ids_cache(self):\n cache = {\n \"dirs\": defaultdict(list),\n \"ids\": {},\n \"root_id\": self._get_item_id(\n self.path,\n use_cache=False,\n hint=\"Confirm the directory exists and you can access it.\",\n ),\n }\n\n self._cache_path_id(self.base, cache[\"root_id\"], cache=cache)\n\n for item in self._gdrive_list(\n \"'{}' in parents and trashed=false\".format(cache[\"root_id\"])\n ):\n item_path = posixpath.join(self.base, item[\"title\"])\n self._cache_path_id(item_path, item[\"id\"], cache=cache)\n\n return cache\n\n def _cache_path_id(self, path, *item_ids, cache=None):\n cache = cache or self._ids_cache\n for item_id in item_ids:\n cache[\"dirs\"][path].append(item_id)\n cache[\"ids\"][item_id] = path\n\n @cached_property\n def _list_params(self):\n params = {\"corpora\": \"default\"}\n if self.root != \"root\" and self.root != \"appDataFolder\":\n drive_id = self._gdrive_shared_drive_id(self.root)\n if drive_id:\n logger.debug(\n \"GDrive remote '{}' is using shared drive id '{}'.\".format(\n self.path, drive_id\n )\n )\n params[\"driveId\"] = drive_id\n params[\"corpora\"] = \"drive\"\n return params\n\n @_gdrive_retry\n def _gdrive_shared_drive_id(self, item_id):\n from pydrive2.files import ApiRequestError\n\n param = {\"id\": item_id}\n # it does not create a file on the remote\n item = self.client.CreateFile(param)\n # ID of the shared drive the item resides in.\n # Only populated for items in shared drives.\n try:\n item.FetchMetadata(\"driveId\")\n except ApiRequestError as exc:\n error_code = exc.error.get(\"code\", 0)\n if error_code == 404:\n raise PermissionError from exc\n raise\n\n return item.get(\"driveId\", None)\n\n def _gdrive_list(self, query):\n param = {\"q\": query, \"maxResults\": 1000}\n param.update(self._list_params)\n file_list = self.client.ListFile(param)\n\n # Isolate and decorate fetching of remote drive items in pages.\n get_list = _gdrive_retry(lambda: next(file_list, None))\n\n # Fetch pages until None is received, lazily flatten the thing.\n return cat(iter(get_list, None))\n\n def _gdrive_list_ids(self, query_ids):\n query = \" or \".join(\n f\"'{query_id}' in parents\" for query_id in query_ids\n )\n query = f\"({query}) and trashed=false\"\n return self._gdrive_list(query)\n\n def _get_remote_item_ids(self, parent_ids, title):\n if not parent_ids:\n return None\n query = \"trashed=false and ({})\".format(\n \" or \".join(\n f\"'{parent_id}' in parents\" for parent_id in parent_ids\n )\n )\n query += \" and title='{}'\".format(title.replace(\"'\", \"\\\\'\"))\n\n # GDrive list API is case insensitive, we need to compare\n # all results and pick the ones with the right title\n return [\n item[\"id\"]\n for item in self._gdrive_list(query)\n if item[\"title\"] == title\n ]\n\n def _get_cached_item_ids(self, path, use_cache):\n if not path:\n return [self.root]\n if use_cache:\n return self._ids_cache[\"dirs\"].get(path, [])\n return []\n\n def _path_to_item_ids(self, path, create=False, use_cache=True):\n item_ids = self._get_cached_item_ids(path, use_cache)\n if item_ids:\n return item_ids\n\n parent_path, title = posixpath.split(path)\n parent_ids = self._path_to_item_ids(parent_path, create, use_cache)\n item_ids = self._get_remote_item_ids(parent_ids, title)\n if item_ids:\n return item_ids\n\n return (\n [self._create_dir(min(parent_ids), title, path)] if create else []\n )\n\n def _get_item_id(self, path, create=False, use_cache=True, hint=None):\n bucket, base = self.split_path(path)\n assert bucket == self.root\n\n item_ids = self._path_to_item_ids(base, create, use_cache)\n if item_ids:\n return min(item_ids)\n\n assert not create\n raise FileNotFoundError(\n errno.ENOENT, os.strerror(errno.ENOENT), hint or path\n )\n\n @_gdrive_retry\n def _gdrive_create_dir(self, parent_id, title):\n parent = {\"id\": parent_id}\n item = self.client.CreateFile(\n {\"title\": title, \"parents\": [parent], \"mimeType\": FOLDER_MIME_TYPE}\n )\n item.Upload()\n return item\n\n @wrap_with(threading.RLock())\n def _create_dir(self, parent_id, title, remote_path):\n cached = self._ids_cache[\"dirs\"].get(remote_path)\n if cached:\n return cached[0]\n\n item = self._gdrive_create_dir(parent_id, title)\n\n if parent_id == self._ids_cache[\"root_id\"]:\n self._cache_path_id(remote_path, item[\"id\"])\n\n return item[\"id\"]\n\n def exists(self, path):\n try:\n self._get_item_id(path)\n except FileNotFoundError:\n return False\n else:\n return True\n\n @_gdrive_retry\n def info(self, path):\n bucket, base = self.split_path(path)\n item_id = self._get_item_id(path)\n gdrive_file = self.client.CreateFile({\"id\": item_id})\n gdrive_file.FetchMetadata()\n\n metadata = {\"name\": posixpath.join(bucket, base.rstrip(\"/\"))}\n if gdrive_file[\"mimeType\"] == FOLDER_MIME_TYPE:\n metadata[\"type\"] = \"directory\"\n metadata[\"size\"] = 0\n metadata[\"name\"] += \"/\"\n else:\n metadata[\"type\"] = \"file\"\n metadata[\"size\"] = int(gdrive_file.get(\"fileSize\"))\n metadata[\"checksum\"] = gdrive_file[\"md5Checksum\"]\n return metadata\n\n def ls(self, path, detail=False):\n bucket, base = self.split_path(path)\n\n cached = base in self._ids_cache[\"dirs\"]\n if cached:\n dir_ids = self._ids_cache[\"dirs\"][base]\n else:\n dir_ids = self._path_to_item_ids(base)\n\n if not dir_ids:\n return None\n\n root_path = posixpath.join(bucket, base)\n contents = []\n for item in self._gdrive_list_ids(dir_ids):\n item_path = posixpath.join(root_path, item[\"title\"])\n if item[\"mimeType\"] == FOLDER_MIME_TYPE:\n contents.append(\n {\n \"type\": \"directory\",\n \"name\": item_path.rstrip(\"/\") + \"/\",\n \"size\": 0,\n }\n )\n else:\n contents.append(\n {\n \"type\": \"file\",\n \"name\": item_path,\n \"size\": int(item[\"fileSize\"]),\n \"checksum\": item[\"md5Checksum\"],\n }\n )\n\n if not cached:\n self._cache_path_id(root_path, *dir_ids)\n\n if detail:\n return contents\n else:\n return [content[\"name\"] for content in contents]\n\n def find(self, path, detail=False, **kwargs):\n bucket, base = self.split_path(path)\n\n seen_paths = set()\n dir_ids = [self._ids_cache[\"ids\"].copy()]\n contents = []\n while dir_ids:\n query_ids = {\n dir_id: dir_name\n for dir_id, dir_name in dir_ids.pop().items()\n if posixpath.commonpath([base, dir_name]) == base\n if dir_id not in seen_paths\n }\n if not query_ids:\n continue\n\n seen_paths |= query_ids.keys()\n\n new_query_ids = {}\n dir_ids.append(new_query_ids)\n for item in self._gdrive_list_ids(query_ids):\n parent_id = item[\"parents\"][0][\"id\"]\n item_path = posixpath.join(query_ids[parent_id], item[\"title\"])\n if item[\"mimeType\"] == FOLDER_MIME_TYPE:\n new_query_ids[item[\"id\"]] = item_path\n self._cache_path_id(item_path, item[\"id\"])\n continue\n\n contents.append(\n {\n \"name\": posixpath.join(bucket, item_path),\n \"type\": \"file\",\n \"size\": int(item[\"fileSize\"]),\n \"checksum\": item[\"md5Checksum\"],\n }\n )\n\n if detail:\n return {content[\"name\"]: content for content in contents}\n else:\n return [content[\"name\"] for content in contents]\n\n def upload_fobj(self, stream, rpath, callback=None, **kwargs):\n parent_id = self._get_item_id(self._parent(rpath), create=True)\n if callback:\n stream = CallbackIOWrapper(\n callback.relative_update, stream, \"read\"\n )\n return self.gdrive_upload_fobj(\n posixpath.basename(rpath.rstrip(\"/\")), parent_id, stream\n )\n\n def put_file(self, lpath, rpath, callback=None, **kwargs):\n if callback:\n callback.set_size(os.path.getsize(lpath))\n with open(lpath, \"rb\") as stream:\n self.upload_fobj(stream, rpath, callback=callback)\n\n @_gdrive_retry\n def gdrive_upload_fobj(self, title, parent_id, stream, callback=None):\n item = self.client.CreateFile(\n {\"title\": title, \"parents\": [{\"id\": parent_id}]}\n )\n item.content = stream\n item.Upload()\n return item\n\n def cp_file(self, lpath, rpath, **kwargs):\n \"\"\"In-memory streamed copy\"\"\"\n with self.open(lpath) as stream:\n # IterStream objects doesn't support full-length\n # seek() calls, so we have to wrap the data with\n # an external buffer.\n buffer = io.BytesIO(stream.read())\n self.upload_fobj(buffer, rpath)\n\n def get_file(self, lpath, rpath, callback=None, block_size=None, **kwargs):\n item_id = self._get_item_id(lpath)\n return self.gdrive_get_file(\n item_id, rpath, callback=callback, block_size=block_size\n )\n\n @_gdrive_retry\n def gdrive_get_file(self, item_id, rpath, callback=None, block_size=None):\n param = {\"id\": item_id}\n # it does not create a file on the remote\n gdrive_file = self.client.CreateFile(param)\n\n extra_args = {}\n if block_size:\n extra_args[\"chunksize\"] = block_size\n\n if callback:\n\n def cb(value, _):\n callback.absolute_update(value)\n\n gdrive_file.FetchMetadata(fields=\"fileSize\")\n callback.set_size(int(gdrive_file.get(\"fileSize\")))\n extra_args[\"callback\"] = cb\n\n gdrive_file.GetContentFile(rpath, **extra_args)\n\n def _open(self, path, mode, **kwargs):\n assert mode in {\"rb\", \"wb\"}\n if mode == \"wb\":\n return GDriveBufferedWriter(self, path)\n else:\n item_id = self._get_item_id(path)\n return self.gdrive_open_file(item_id)\n\n @_gdrive_retry\n def gdrive_open_file(self, item_id):\n param = {\"id\": item_id}\n # it does not create a file on the remote\n gdrive_file = self.client.CreateFile(param)\n fd = gdrive_file.GetContentIOBuffer()\n return IterStream(iter(fd))\n\n def rm_file(self, path):\n item_id = self._get_item_id(path)\n self.gdrive_delete_file(item_id)\n\n @_gdrive_retry\n def gdrive_delete_file(self, item_id):\n from pydrive2.files import ApiRequestError\n\n param = {\"id\": item_id}\n # it does not create a file on the remote\n item = self.client.CreateFile(param)\n\n try:\n item.Trash() if self._trash_only else item.Delete()\n except ApiRequestError as exc:\n http_error_code = exc.error.get(\"code\", 0)\n if (\n http_error_code == 403\n and self._list_params[\"corpora\"] == \"drive\"\n and exc.GetField(\"location\") == \"file.permissions\"\n ):\n raise PermissionError(\n \"Insufficient permissions to {}. You should have {} \"\n \"access level for the used shared drive. More details \"\n \"at {}.\".format(\n \"move the file into Trash\"\n if self._trash_only\n else \"permanently delete the file\",\n \"Manager or Content Manager\"\n if self._trash_only\n else \"Manager\",\n \"https://support.google.com/a/answer/7337554\",\n )\n ) from exc\n raise\n\n\nclass GDriveBufferedWriter(io.IOBase):\n def __init__(self, fs, path):\n self.fs = fs\n self.path = path\n self.buffer = io.BytesIO()\n self._closed = False\n\n def write(self, *args, **kwargs):\n self.buffer.write(*args, **kwargs)\n\n def readable(self):\n return False\n\n def writable(self):\n return not self.readable()\n\n def flush(self):\n self.buffer.flush()\n try:\n self.fs.upload_fobj(self.buffer, self.path)\n finally:\n self._closed = True\n\n def close(self):\n if self._closed:\n return None\n\n self.flush()\n self.buffer.close()\n self._closed = True\n\n def __enter__(self):\n return self\n\n def __exit__(self, *exc_info):\n self.close()\n\n @property\n def closed(self):\n return self._closed\n" }, { "path": "DriveDownloader/pydrive2/fs/utils.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nimport io\n\n\nclass IterStream(io.RawIOBase):\n \"\"\"Wraps an iterator yielding bytes as a file object\"\"\"\n\n def __init__(self, iterator): # pylint: disable=super-init-not-called\n self.iterator = iterator\n self.leftover = b\"\"\n\n def readable(self):\n return True\n\n def writable(self) -> bool:\n return False\n\n # Python 3 requires only .readinto() method, it still uses other ones\n # under some circumstances and falls back if those are absent. Since\n # iterator already constructs byte strings for us, .readinto() is not the\n # most optimal, so we provide .read1() too.\n\n def readinto(self, b):\n try:\n n = len(b) # We're supposed to return at most this much\n chunk = self.leftover or next(self.iterator)\n output, self.leftover = chunk[:n], chunk[n:]\n\n n_out = len(output)\n b[:n_out] = output\n return n_out\n except StopIteration:\n return 0 # indicate EOF\n\n readinto1 = readinto\n\n def read1(self, n=-1):\n try:\n chunk = self.leftover or next(self.iterator)\n except StopIteration:\n return b\"\"\n\n # Return an arbitrary number or bytes\n if n <= 0:\n self.leftover = b\"\"\n return chunk\n\n output, self.leftover = chunk[:n], chunk[n:]\n return output\n\n def peek(self, n):\n while len(self.leftover) < n:\n try:\n self.leftover += next(self.iterator)\n except StopIteration:\n break\n return self.leftover[:n]\n" }, { "path": "DriveDownloader/pydrive2/settings.py", "content": "# Credit: https://github.com/iterative/PyDrive2\n\nfrom yaml import load\nfrom yaml import YAMLError\nimport os\n\ntry:\n from yaml import CLoader as Loader\nexcept ImportError:\n from yaml import Loader\n\nSETTINGS_FILE = \"settings.yaml\"\nSETTINGS_STRUCT = {\n \"client_config_backend\": {\n \"type\": str,\n \"required\": True,\n \"default\": \"file\",\n \"dependency\": [\n {\"value\": \"file\", \"attribute\": [\"client_config_file\"]},\n {\"value\": \"settings\", \"attribute\": [\"client_config\"]},\n {\"value\": \"service\", \"attribute\": [\"service_config\"]},\n ],\n },\n \"save_credentials\": {\n \"type\": bool,\n \"required\": True,\n \"default\": False,\n \"dependency\": [\n {\"value\": True, \"attribute\": [\"save_credentials_backend\"]}\n ],\n },\n \"get_refresh_token\": {\"type\": bool, \"required\": False, \"default\": False},\n \"client_config_file\": {\n \"type\": str,\n \"required\": False,\n \"default\": \"client_secrets.json\",\n },\n \"save_credentials_backend\": {\n \"type\": str,\n \"required\": False,\n \"dependency\": [\n {\"value\": \"file\", \"attribute\": [\"save_credentials_file\"]}\n ],\n },\n \"client_config\": {\n \"type\": dict,\n \"required\": False,\n \"struct\": {\n \"client_id\": {\"type\": str, \"required\": True},\n \"client_secret\": {\"type\": str, \"required\": True},\n \"auth_uri\": {\n \"type\": str,\n \"required\": True,\n \"default\": \"https://accounts.google.com/o/oauth2/auth\",\n },\n \"token_uri\": {\n \"type\": str,\n \"required\": True,\n \"default\": \"https://accounts.google.com/o/oauth2/token\",\n },\n \"redirect_uri\": {\n \"type\": str,\n \"required\": True,\n \"default\": \"urn:ietf:wg:oauth:2.0:oob\",\n },\n \"revoke_uri\": {\"type\": str, \"required\": True, \"default\": None},\n },\n },\n \"service_config\": {\n \"type\": dict,\n \"required\": False,\n \"struct\": {\n \"client_user_email\": {\n \"type\": str,\n \"required\": True,\n \"default\": None,\n },\n \"client_service_email\": {\"type\": str, \"required\": False},\n \"client_pkcs12_file_path\": {\"type\": str, \"required\": False},\n \"client_json_file_path\": {\"type\": str, \"required\": False},\n },\n },\n \"oauth_scope\": {\n \"type\": list,\n \"required\": True,\n \"struct\": str,\n \"default\": [\"https://www.googleapis.com/auth/drive\"],\n },\n \"save_credentials_file\": {\"type\": str, \"required\": False, \"default\": os.path.join(os.environ['HOME'], '.credentials.json')},\n}\n\n\nclass SettingsError(IOError):\n \"\"\"Error while loading/saving settings\"\"\"\n\n\nclass InvalidConfigError(IOError):\n \"\"\"Error trying to read client configuration.\"\"\"\n\n\ndef LoadSettingsFile(filename=SETTINGS_FILE):\n \"\"\"Loads settings file in yaml format given file name.\n\n :param filename: path for settings file. 'settings.yaml' by default.\n :type filename: str.\n :raises: SettingsError\n \"\"\"\n try:\n with open(filename, \"r\") as stream:\n data = load(stream, Loader=Loader)\n except (YAMLError, IOError) as e:\n raise SettingsError(e)\n return data\n\n\ndef ValidateSettings(data):\n \"\"\"Validates if current settings is valid.\n\n :param data: dictionary containing all settings.\n :type data: dict.\n :raises: InvalidConfigError\n \"\"\"\n _ValidateSettingsStruct(data, SETTINGS_STRUCT)\n\n\ndef _ValidateSettingsStruct(data, struct):\n \"\"\"Validates if provided data fits provided structure.\n\n :param data: dictionary containing settings.\n :type data: dict.\n :param struct: dictionary containing structure information of settings.\n :type struct: dict.\n :raises: InvalidConfigError\n \"\"\"\n # Validate required elements of the setting.\n for key in struct:\n if struct[key][\"required\"]:\n _ValidateSettingsElement(data, struct, key)\n\n\ndef _ValidateSettingsElement(data, struct, key):\n \"\"\"Validates if provided element of settings data fits provided structure.\n\n :param data: dictionary containing settings.\n :type data: dict.\n :param struct: dictionary containing structure information of settings.\n :type struct: dict.\n :param key: key of the settings element to validate.\n :type key: str.\n :raises: InvalidConfigError\n \"\"\"\n # Check if data exists. If not, check if default value exists.\n value = data.get(key)\n data_type = struct[key][\"type\"]\n if value is None:\n try:\n default = struct[key][\"default\"]\n except KeyError:\n raise InvalidConfigError(\"Missing required setting %s\" % key)\n else:\n data[key] = default\n # If data exists, Check type of the data\n elif type(value) is not data_type:\n raise InvalidConfigError(\n \"Setting %s should be type %s\" % (key, data_type)\n )\n # If type of this data is dict, check if structure of the data is valid.\n if data_type is dict:\n _ValidateSettingsStruct(data[key], struct[key][\"struct\"])\n # If type of this data is list, check if all values in the list is valid.\n elif data_type is list:\n for element in data[key]:\n if type(element) is not struct[key][\"struct\"]:\n raise InvalidConfigError(\n \"Setting %s should be list of %s\"\n % (key, struct[key][\"struct\"])\n )\n # Check dependency of this attribute.\n dependencies = struct[key].get(\"dependency\")\n if dependencies:\n for dependency in dependencies:\n if value == dependency[\"value\"]:\n for reqkey in dependency[\"attribute\"]:\n _ValidateSettingsElement(data, struct, reqkey)\n" }, { "path": "DriveDownloader/utils/__init__.py", "content": "from .misc import *\nfrom .multithread import MultiThreadDownloader" }, { "path": "DriveDownloader/utils/misc.py", "content": "#############################################\r\n# Author: Hongwei Fan #\r\n# E-mail: hwnorm@outlook.com #\r\n# Homepage: https://github.com/hwfan #\r\n#############################################\r\nfrom urllib.parse import urlparse\r\n\r\ndef format_size(value):\r\n units = [\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\"]\r\n size = 1024.0\r\n for i in range(len(units)):\r\n if (value / size) < 1:\r\n return \"%.2f %s\" % (value, units[i])\r\n value = value / size\r\n return value\r\n\r\ndef judge_session(url):\r\n if '1drv.ms' in url or '1drv.ws' in url:\r\n return 'OneDrive'\r\n elif 'drive.google.com' in url:\r\n return 'GoogleDrive'\r\n elif 'sharepoint' in url:\r\n return 'SharePoint'\r\n elif 'dropbox' in url:\r\n return 'DropBox'\r\n else:\r\n return 'DirectLink'\r\n\r\ndef judge_scheme(url):\r\n return urlparse(url).scheme" }, { "path": "DriveDownloader/utils/multithread.py", "content": "import copy\nimport threading\nimport shutil\nimport os\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\ndef download_session(session_func, url, filename, proc_id, start, end, used_proxy, progress_bar, force_back_google):\n drive_session = session_func(used_proxy)\n drive_session.set_range(start, end)\n drive_session.connect(url, filename, proc_id=proc_id, force_backup=force_back_google)\n interrupted = drive_session.save_response_content(start=int(start), end=int(end), proc_id=proc_id, progress_bar=progress_bar)\n return interrupted\n\nclass MultiThreadDownloader:\n def __init__(self, progress_bar, session_func, used_proxy, filesize, thread_number):\n self.progress = progress_bar\n self.session_func = session_func\n self.used_proxy = used_proxy\n self.thread_number = thread_number\n self.filesize = filesize\n self.get_ranges()\n\n def get_ranges(self):\n self.ranges = []\n offset = int(self.filesize / self.thread_number)\n for i in range(self.thread_number):\n if i == self.thread_number - 1:\n self.ranges.append((str(i * offset), str(int(self.filesize-1))))\n else:\n self.ranges.append((str(i * offset), str((i+1) * offset - 1)))\n \n def get(self, url, filename, force_back_google):\n with self.progress:\n with ThreadPoolExecutor(max_workers=len(self.ranges)) as pool:\n ts = []\n status = []\n for proc_id, each_range in enumerate(self.ranges):\n start, end = each_range\n task_id = self.progress.add_task(\"download\", filename=filename, proc_id=proc_id, start=False)\n t = pool.submit(download_session, self.session_func, url, filename, proc_id, start, end, self.used_proxy, self.progress, force_back_google)\n ts.append(t)\n for t in as_completed(ts):\n interrupted = t.result()\n status.append(interrupted)\n if True in status:\n return True\n return False\n\n def concatenate(self, filename):\n sub_filenames = []\n dirname = os.path.dirname(filename)\n tmp_dirname = os.path.join(dirname, 'tmp')\n for proc_id in range(len(self.ranges)):\n name, ext = os.path.splitext(filename)\n name = name + '_{}'.format(proc_id)\n sub_filename = name + ext\n sub_basename = os.path.basename(sub_filename)\n sub_filename = os.path.join(tmp_dirname, sub_basename)\n sub_filenames.append(sub_filename)\n\n with open(filename, 'wb') as wfd:\n for f in sub_filenames:\n with open(f, 'rb') as fd:\n shutil.copyfileobj(fd, wfd)\n os.remove(f)\n shutil.rmtree(tmp_dirname)\n " }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2020 Hongwei Fan\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# DriveDownloader\n\nEnglish | [中文文档](README_CN.md)\n\n**DriveDownloader** is a Python-based **CLI** tool for downloading files on online drives. With DriveDownloader, one can download the resources from netdrive with **only one command line**. \n\nDriveDownloader now supports:\n - OneDrive\n - OneDrive for Business\n - GoogleDrive\n - Dropbox\n - Direct Link\n\n## Usage\n\n ```\n ddl URL/FILELIST [--filename FILENAME] [--thread-number NUMBER] [--version] [--help]\n ```\n\n - `URL/FILELIST`: target url/filelist to download from. **The example of filelist is shown in `tests/test.list`.**\n - `--filename/-o FILENAME`: (optional) output filename. Example: 'hello.txt'\n - `--thread-number/-n NUMBER`: (optional) the thread number when using multithread.\n - `--force-back-google/-F`: (optional) use the backup downloader for Google drive (it needs authentication, but is more stable).\n - Using proxy:\n - Set the environment variables `http_proxy` and `https_proxy` to your proxy addresses, and DriveDownloader will automatically read them.\n - Resume:\n - If your download was interrupted accidentally, simply restart the command will resume, regardless the number of threads.\n \n## Installation\n 1. Install from pip\n ```\n pip install DriveDownloader\n ```\n\n 2. Install from source\n ```\n git clone https://github.com/hwfan/DriveDownloader.git && cd DriveDownloader\n python setup.py install\n ```\n\n## Quick Start\n \n Coming Soon.\n\n## Requirements\n\n - Python 3.7+\n - Use `pip install -r requirements.txt` to install the packages.\n - Proxy server if necessary. **We don't provide proxy service for DriveDownloader.**\n \n## Examples\n\n You can also see these examples in `tests/run.sh`.\n\n ```\n echo \"Unit Tests of DriveDownloader\"\n mkdir -p test_outputs\n\n echo \"Testing Direct Link...\"\n # direct link\n ddl https://www.google.com.hk/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png -o test_outputs/directlink.png\n\n echo \"Testing OneDrive...\"\n # OneDrive\n ddl https://1drv.ms/t/s!ArUVoRxpBphY5U-a3JznLkLG1uEY?e=czbq1R -o test_outputs/hello_od.txt\n\n echo \"Testing GoogleDrive...\"\n # GoogleDrive\n ddl https://drive.google.com/file/d/1XQRdK8ewbpOlQn7CvB99aT1FLi6cUKt_/view?usp=sharing -o test_outputs/hello_gd.txt\n\n echo \"Testing SharePoint...\"\n # SharePoint\n ddl https://bupteducn-my.sharepoint.com/:t:/g/personal/hwfan_bupt_edu_cn/EQzn4SeFkJZHq8OikhX7X3QB97PSiNvJpPVtllBQln8EQw?e=NmgRSc -o test_outputs/hello_sp.txt\n\n echo \"Testing Dropbox...\"\n # Dropbox\n ddl https://www.dropbox.com/s/bd0bak3h9dlfw3z/hello.txt?dl=0 -o test_outputs/hello_db.txt\n\n echo \"Testing File List...\"\n # file list\n ddl test.list -l\n\n echo \"Testing Multi Thread...\"\n # Multi Thread\n ddl https://www.dropbox.com/s/r4bme0kew42oo7e/Get%20Started%20with%20Dropbox.pdf?dl=0 -o test_outputs/Dropbox.pdf -n 8\n ```\n\n## FAQ\n\n- Why does \"Size:Invalid\" occur?\n\n - We extract the size of file from the \"Content-Length\" of HTTP response. If this parameter is empty, the file size will fall back to \"Invalid\". (The response of GoogleDrive often hides this header.)\n\n- I couldn't connect to the target server through a socks5 proxy.\n\n - Try \"socks5h\" as the protocol prefix instead. It will transmit the url to proxy server for parsing.\n\n- There exists some old bugs in my DriveDownloader.\n\n - Try `pip install DriveDownloader --force-reinstall --upgrade` to update. We keep the latest version of DDL free from those bugs.\n\n- !{some string}: event not found\n\n - Since bash can parse \"!\" from the url, single quotes(') should be added before and after the url when using bash.\n \n ```\n ddl 'https://1drv.ms/t/s!ArUVoRxpBphY5U-a3JznLkLG1uEY?e=czbq1R' -o test_outputs/hello_od.txt\n ```\n\n## Acknowledgement\n\nSome code of DriveDownloader is borrowed from [PyDrive2](https://github.com/iterative/PyDrive2) and [rich](https://github.com/Textualize/rich). Thanks for their wonderful jobs!\n\n## TODOs\n\n - [x] General downloader API - one class for downloading, and several inheritance classes to load the configurations.\n - [x] Support more netdrives - OneDrive for Business, Dropbox, ...\n - [x] Downloading files from a list.\n - [x] Multi-thread downloading.\n - [x] Resume downloading.\n - [ ] Folder downloading.\n - [ ] Window-based UI.\n - [ ] Quick Start.\n\n## Update Log\n\n### v1.6.0\n\n- Added automatic resume downloading.\n- Changed the progress bar manager to [rich](https://github.com/Textualize/rich).\n\n### v1.5.0\n\n- Solved the problem of \"not accessible\" when downloading a large file on Google Drive.\n- The input type (URL/FILELIST) is now automatically detected by the downloader, and `-l/--list` is deprecated.\n- The proxy server is now parsed from environmental variables, and `-p/--proxy` is deprecated.\n- Added the version option `-v/--version`.\n\n### v1.4.0\n\n- Supported Multi-thread and downloading from a list and a direct link.\n- Removed interactive mode.\n\n### v1.3.0\n\n- Supported Sharepoint and Dropbox.\n- Removed the deprecated fake-useragent.\n" }, { "path": "README_CN.md", "content": "# DriveDownloader\n\n[English](README.md) | 中文文档\n\n**DriveDownloader**是一个基于Python的命令行工具，用来下载OneDrive, 谷歌网盘等在线存储上的文件。使用DriveDownloader，只需要一行简洁的命令，就可以从各类网盘上下载文件。\n\nDriveDownloader当前支持：\n - OneDrive\n - OneDrive for Business\n - GoogleDrive\n - Dropbox\n - 直链\n\n## 命令用法\n\n ```\n ddl URL/FILELIST [--filename FILENAME] [--thread-number NUMBER] [--force-back-google] [--version] [--help]\n ```\n\n - `URL/FILELIST`: 目标的URL或者文件列表。**文件列表的格式请参考：`tests/test.list`.**\n - `--filename/-o FILENAME`: (可选) 输出的文件名，如'hello.txt'\n - `--thread-number/-n NUMBER`: (可选) 多线程的线程数量。\n - `--force-back-google/-F`: (可选) 对于谷歌网盘使用备份下载器 (需要谷歌账号认证，但可以保证稳定连接)\n - 使用代理服务器：\n - 请将环境变量 `http_proxy` 与 `https_proxy` 设置成你的代理服务器地址，DriveDownloader会自动读取它们。\n - 断点续传:\n - 如果您的下载意外中断，只需要重启同样的命令即可恢复。\n \n## 安装方式\n\n 1. 从pip安装\n ```\n pip install DriveDownloader\n ```\n\n 2. 从源代码安装\n ```\n git clone https://github.com/hwfan/DriveDownloader.git && cd DriveDownloader\n python setup.py install\n ```\n\n## 快速开始\n \n 制作中，与新版本共同发布。\n\n## 依赖\n\n - Python 3.7+\n - 请使用`pip install -r requirements.txt`安装依赖。\n \n## 用例\n\n 这些用例也可以在`tests/run.sh`中找到。\n\n ```\n echo \"Unit Tests of DriveDownloader\"\n mkdir -p test_outputs\n\n echo \"Testing Direct Link...\"\n # direct link\n ddl https://www.google.com.hk/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png -o test_outputs/directlink.png\n\n echo \"Testing OneDrive...\"\n # OneDrive\n ddl https://1drv.ms/t/s!ArUVoRxpBphY5U-a3JznLkLG1uEY?e=czbq1R -o test_outputs/hello_od.txt\n\n echo \"Testing GoogleDrive...\"\n # GoogleDrive\n ddl https://drive.google.com/file/d/1XQRdK8ewbpOlQn7CvB99aT1FLi6cUKt_/view?usp=sharing -o test_outputs/hello_gd.txt\n\n echo \"Testing SharePoint...\"\n # SharePoint\n ddl https://bupteducn-my.sharepoint.com/:t:/g/personal/hwfan_bupt_edu_cn/EQzn4SeFkJZHq8OikhX7X3QB97PSiNvJpPVtllBQln8EQw?e=NmgRSc -o test_outputs/hello_sp.txt\n\n echo \"Testing Dropbox...\"\n # Dropbox\n ddl https://www.dropbox.com/s/bd0bak3h9dlfw3z/hello.txt?dl=0 -o test_outputs/hello_db.txt\n\n echo \"Testing File List...\"\n # file list\n ddl test.list -l\n\n echo \"Testing Multi Thread...\"\n # Multi Thread\n ddl https://www.dropbox.com/s/r4bme0kew42oo7e/Get%20Started%20with%20Dropbox.pdf?dl=0 -o test_outputs/Dropbox.pdf -n 8\n ```\n\n## 常见问题\n\n- 为什么提示\"Size:Invalid\"?\n\n - 我们根据HTTP报文中的\"Content-Length\"提取文件大小。如果该参数置空，文件大小就会回落至默认的\"Invalid\". (谷歌网盘会隐藏这个参数)\n\n- 通过socks5代理无法连接目标服务器。\n\n - 请使用\"socks5h\"作为协议前缀。该前缀会将URL发送给代理服务器进行解析。\n\n- 我的DriveDownloader中有一些没有修复的bug。\n\n - 请使用`pip install DriveDownloader --force-reinstall --upgrade`更新DriveDownloader。\n\n- !{some string}: event not found\n\n - 在bash中，URL中的\"!\"是一个关键字, 请在URL前后增加引号(')以解决该问题，例如\n \n ```\n ddl 'https://1drv.ms/t/s!ArUVoRxpBphY5U-a3JznLkLG1uEY?e=czbq1R' -o test_outputs/hello_od.txt\n ```\n\n## 鸣谢\n\n本项目的部分代码来源于[PyDrive2](https://github.com/iterative/PyDrive2)与[rich](https://github.com/Textualize/rich)。感谢他们优秀的工作！\n\n## 开发计划\n\n - [x] 通用的下载API - 一个下载类，多个网盘下载的继承类。\n - [x] 支持更多网盘 - OneDrive for Business, Dropbox, 直链等。\n - [x] 从列表下载\n - [x] 多线程下载\n - [x] 断点续传\n - [ ] 基于窗口的UI\n - [ ] 快速开始\n \n## 更新日志\n\n### v1.6.0\n\n- 增加断点续传功能。\n- 采用新的进度条管理器[rich](https://github.com/Textualize/rich)。\n\n### v1.5.0\n\n- 解决了在部分情况下无法访问谷歌网盘文件的问题。\n- 输入是URL还是文件将由下载器自行判断，`-l/--list`选项将不再维护。\n- 统一读取环境变量中的代理，`-p/--proxy`选项将不再维护。\n- 增加了版本号显示选项`-v/--version`。\n\n### v1.4.0\n\n- 支持多线程，从文件下载，从直链下载。\n- 移除了交互模式。\n\n### v1.3.0\n\n- 支持Sharepoint与Dropbox。\n- 移除了fake-useragent的依赖。" }, { "path": "requirements.txt", "content": "argparse\r\nrequests\r\ntqdm\r\nrich\r\npysocks\r\nrequests_random_user_agent\r\ngoogle-api-python-client >= 1.12.5\r\nsix >= 1.13.0\r\noauth2client >= 4.0.0\r\nPyYAML >= 3.0\r\npyOpenSSL >= 19.1.0" }, { "path": "setup.py", "content": "from setuptools import setup, find_packages\r\nsetup(\r\n name = \"DriveDownloader\",\r\n version = \"1.6.0.post1\",\r\n keywords = (\"drivedownloader\", \"drive\", \"netdrive\", \"download\"),\r\n description = \"A Python netdrive downloader.\",\r\n long_description = \"A Python netdrive downloader.\",\r\n license = \"MIT Licence\",\r\n\r\n url = \"https://github.com/hwfan\",\r\n author = \"hwfan\",\r\n author_email = \"hwnorm@outlook.com\",\r\n\r\n packages = find_packages(),\r\n include_package_data = True,\r\n platforms = \"any\",\r\n install_requires = ['argparse', 'requests', 'tqdm', 'rich', 'pysocks', 'requests_random_user_agent',\r\n \"google-api-python-client >= 1.12.5\", \"six >= 1.13.0\", \"oauth2client >= 4.0.0\",\r\n \"PyYAML >= 3.0\", \"pyOpenSSL >= 19.1.0\"],\r\n scripts = [],\r\n entry_points = {\r\n 'console_scripts': [\r\n 'ddl = DriveDownloader.downloader:simple_cli'\r\n ]\r\n }\r\n)\r\n" }, { "path": "tests/run.sh", "content": "echo \"Unit Tests of DriveDownloader\"\nmkdir -p test_outputs\n\necho \"Testing Direct Link...\"\n# direct link\nddl https://www.google.com.hk/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png -o test_outputs/directlink.png\n\necho \"Testing OneDrive...\"\n# OneDrive\nddl https://1drv.ms/t/s!ArUVoRxpBphY5U-a3JznLkLG1uEY?e=czbq1R -o test_outputs/hello_od.txt\n\necho \"Testing GoogleDrive...\"\n# GoogleDrive\nddl https://drive.google.com/file/d/1XQRdK8ewbpOlQn7CvB99aT1FLi6cUKt_/view?usp=sharing -o test_outputs/hello_gd.txt\n\necho \"Testing SharePoint...\"\n# SharePoint\nddl https://bupteducn-my.sharepoint.com/:t:/g/personal/hwfan_bupt_edu_cn/EQzn4SeFkJZHq8OikhX7X3QB97PSiNvJpPVtllBQln8EQw?e=NmgRSc -o test_outputs/hello_sp.txt\n\necho \"Testing Dropbox...\"\n# Dropbox\nddl https://www.dropbox.com/s/bd0bak3h9dlfw3z/hello.txt?dl=0 -o test_outputs/hello_db.txt\n\necho \"Testing File List...\"\n# file list\nddl test.list\n\necho \"Testing Multi Thread...\"\n# Multi Thread\nddl https://www.dropbox.com/s/r4bme0kew42oo7e/Get%20Started%20with%20Dropbox.pdf?dl=0 -o test_outputs/Dropbox.pdf -n 8" }, { "path": "tests/test.list", "content": "https://drive.google.com/file/d/1XQRdK8ewbpOlQn7CvB99aT1FLi6cUKt_/view?usp=sharing test_outputs/list_hello_google.txt\nhttps://www.dropbox.com/s/bd0bak3h9dlfw3z/hello.txt?dl=0 test_outputs/list_hello_dropbox.txt\n" } ]