![\"Download](\"https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png\"){kind=icon}
](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile)\n - APK download: https://github.com/canewsin/zeronet_mobile/releases\n - XDA Labs: https://labs.xda-developers.com/store/app/in.canews.zeronet\n \n#### Docker\nThere is an official image, built from source at: https://hub.docker.com/r/nofish/zeronet/\n\n### Install from source\n\n - `wget https://github.com/HelloZeroNet/ZeroNet/archive/py3/ZeroNet-py3.tar.gz`\n - `tar xvpfz ZeroNet-py3.tar.gz`\n - `cd ZeroNet-py3`\n - `sudo apt-get update`\n - `sudo apt-get install python3-pip`\n - `sudo python3 -m pip install -r requirements.txt`\n - Start with: `python3 zeronet.py`\n - Open the ZeroHello landing page in your browser by navigating to: http://127.0.0.1:43110/\n\n## Current limitations\n\n* ~~No torrent-like file splitting for big file support~~ (big file support added)\n* ~~No more anonymous than Bittorrent~~ (built-in full Tor support added)\n* File transactions are not compressed ~~or encrypted yet~~ (TLS encryption added)\n* No private sites\n\n\n## How can I create a ZeroNet site?\n\n * Click on **⋮** > **\"Create new, empty site\"** menu item on the site [ZeroHello](http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D).\n * You will be **redirected** to a completely new site that is only modifiable by you!\n * You can find and modify your site's content in **data/[yoursiteaddress]** directory\n * After the modifications open your site, drag the topright \"0\" button to left, then press **sign** and **publish** buttons on the bottom\n\nNext steps: [ZeroNet Developer Documentation](https://zeronet.io/docs/site_development/getting_started/)\n\n## Help keep this project alive\n\n- Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX\n- Paypal: https://zeronet.io/docs/help_zeronet/donate/\n\n### Sponsors\n\n* Better macOS/Safari compatibility made possible by [BrowserStack.com](https://www.browserstack.com)\n\n#### Thank you!\n\n* More info, help, changelog, zeronet sites: https://www.reddit.com/r/zeronet/\n* Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) or on [gitter](https://gitter.im/HelloZeroNet/ZeroNet)\n* Email: hello@zeronet.io (PGP: [960F FF2D 6C14 5AA6 13E8 491B 5B63 BAE6 CB96 13AE](https://zeronet.io/files/tamas@zeronet.io_pub.asc))\n"
},
{
"path": "Vagrantfile",
"content": "# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\nVAGRANTFILE_API_VERSION = \"2\"\n\nVagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n\n #Set box\n config.vm.box = \"ubuntu/trusty64\"\n\n #Do not check fo updates\n config.vm.box_check_update = false\n\n #Add private network\n config.vm.network \"private_network\", type: \"dhcp\"\n\n #Redirect ports\n config.vm.network \"forwarded_port\", guest: 43110, host: 43110\n config.vm.network \"forwarded_port\", guest: 15441, host: 15441\n\n #Sync folder using NFS if not windows\n config.vm.synced_folder \".\", \"/vagrant\",\n :nfs => !Vagrant::Util::Platform.windows?\n\n #Virtal Box settings\n config.vm.provider \"virtualbox\" do |vb|\n # Don't boot with headless mode\n #vb.gui = true\n\n # Set VM settings\n vb.customize [\"modifyvm\", :id, \"--memory\", \"512\"]\n vb.customize [\"modifyvm\", :id, \"--cpus\", 1]\n end\n\n #Update system\n config.vm.provision \"shell\",\n inline: \"sudo apt-get update -y && sudo apt-get upgrade -y\"\n\n #Install deps\n config.vm.provision \"shell\",\n inline: \"sudo apt-get install msgpack-python python-gevent python-pip python-dev -y\"\n config.vm.provision \"shell\",\n inline: \"sudo pip install msgpack --upgrade\"\n\nend\n"
},
{
"path": "plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py",
"content": "import time\nimport urllib.request\nimport struct\nimport socket\n\nimport lib.bencode_open as bencode_open\nfrom lib.subtl.subtl import UdpTrackerClient\nimport socks\nimport sockshandler\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom Debug import Debug\nfrom util import helper\n\n\n# We can only import plugin host clases after the plugins are loaded\n@PluginManager.afterLoad\ndef importHostClasses():\n global Peer, AnnounceError\n from Peer import Peer\n from Site.SiteAnnouncer import AnnounceError\n\n\n@PluginManager.registerTo(\"SiteAnnouncer\")\nclass SiteAnnouncerPlugin(object):\n def getSupportedTrackers(self):\n trackers = super(SiteAnnouncerPlugin, self).getSupportedTrackers()\n if config.disable_udp or config.trackers_proxy != \"disable\":\n trackers = [tracker for tracker in trackers if not tracker.startswith(\"udp://\")]\n\n return trackers\n\n def getTrackerHandler(self, protocol):\n if protocol == \"udp\":\n handler = self.announceTrackerUdp\n elif protocol == \"http\":\n handler = self.announceTrackerHttp\n elif protocol == \"https\":\n handler = self.announceTrackerHttps\n else:\n handler = super(SiteAnnouncerPlugin, self).getTrackerHandler(protocol)\n return handler\n\n def announceTrackerUdp(self, tracker_address, mode=\"start\", num_want=10):\n s = time.time()\n if config.disable_udp:\n raise AnnounceError(\"Udp disabled by config\")\n if config.trackers_proxy != \"disable\":\n raise AnnounceError(\"Udp trackers not available with proxies\")\n\n ip, port = tracker_address.split(\"/\")[0].split(\":\")\n tracker = UdpTrackerClient(ip, int(port))\n if helper.getIpType(ip) in self.getOpenedServiceTypes():\n tracker.peer_port = self.fileserver_port\n else:\n tracker.peer_port = 0\n tracker.connect()\n if not tracker.poll_once():\n raise AnnounceError(\"Could not connect\")\n tracker.announce(info_hash=self.site.address_sha1, num_want=num_want, left=431102370)\n back = tracker.poll_once()\n if not back:\n raise AnnounceError(\"No response after %.0fs\" % (time.time() - s))\n elif type(back) is dict and \"response\" in back:\n peers = back[\"response\"][\"peers\"]\n else:\n raise AnnounceError(\"Invalid response: %r\" % back)\n\n return peers\n\n def httpRequest(self, url):\n headers = {\n 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',\n 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',\n 'Accept-Encoding': 'none',\n 'Accept-Language': 'en-US,en;q=0.8',\n 'Connection': 'keep-alive'\n }\n\n req = urllib.request.Request(url, headers=headers)\n\n if config.trackers_proxy == \"tor\":\n tor_manager = self.site.connection_server.tor_manager\n handler = sockshandler.SocksiPyHandler(socks.SOCKS5, tor_manager.proxy_ip, tor_manager.proxy_port)\n opener = urllib.request.build_opener(handler)\n return opener.open(req, timeout=50)\n elif config.trackers_proxy == \"disable\":\n return urllib.request.urlopen(req, timeout=25)\n else:\n proxy_ip, proxy_port = config.trackers_proxy.split(\":\")\n handler = sockshandler.SocksiPyHandler(socks.SOCKS5, proxy_ip, int(proxy_port))\n opener = urllib.request.build_opener(handler)\n return opener.open(req, timeout=50)\n\n def announceTrackerHttps(self, *args, **kwargs):\n kwargs[\"protocol\"] = \"https\"\n return self.announceTrackerHttp(*args, **kwargs)\n\n def announceTrackerHttp(self, tracker_address, mode=\"start\", num_want=10, protocol=\"http\"):\n tracker_ip, tracker_port = tracker_address.rsplit(\":\", 1)\n if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes():\n port = self.fileserver_port\n else:\n port = 1\n params = {\n 'info_hash': self.site.address_sha1,\n 'peer_id': self.peer_id, 'port': port,\n 'uploaded': 0, 'downloaded': 0, 'left': 431102370, 'compact': 1, 'numwant': num_want,\n 'event': 'started'\n }\n\n url = protocol + \"://\" + tracker_address + \"?\" + urllib.parse.urlencode(params)\n\n s = time.time()\n response = None\n # Load url\n if config.tor == \"always\" or config.trackers_proxy != \"disable\":\n timeout = 60\n else:\n timeout = 30\n\n with gevent.Timeout(timeout, False): # Make sure of timeout\n req = self.httpRequest(url)\n response = req.read()\n req.close()\n req = None\n\n if not response:\n raise AnnounceError(\"No response after %.0fs\" % (time.time() - s))\n\n # Decode peers\n try:\n peer_data = bencode_open.loads(response)[b\"peers\"]\n response = None\n peer_count = int(len(peer_data) / 6)\n peers = []\n for peer_offset in range(peer_count):\n off = 6 * peer_offset\n peer = peer_data[off:off + 6]\n addr, port = struct.unpack('!LH', peer)\n peers.append({\"addr\": socket.inet_ntoa(struct.pack('!L', addr)), \"port\": port})\n except Exception as err:\n raise AnnounceError(\"Invalid response: %r (%s)\" % (response, Debug.formatException(err)))\n\n return peers\n"
},
{
"path": "plugins/AnnounceBitTorrent/__init__.py",
"content": "from . import AnnounceBitTorrentPlugin"
},
{
"path": "plugins/AnnounceBitTorrent/plugin_info.json",
"content": "{\n\t\"name\": \"AnnounceBitTorrent\",\n\t\"description\": \"Discover new peers using BitTorrent trackers.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/AnnounceLocal/AnnounceLocalPlugin.py",
"content": "import time\n\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom . import BroadcastServer\n\n\n@PluginManager.registerTo(\"SiteAnnouncer\")\nclass SiteAnnouncerPlugin(object):\n def announce(self, force=False, *args, **kwargs):\n local_announcer = self.site.connection_server.local_announcer\n\n thread = None\n if local_announcer and (force or time.time() - local_announcer.last_discover > 5 * 60):\n thread = gevent.spawn(local_announcer.discover, force=force)\n back = super(SiteAnnouncerPlugin, self).announce(force=force, *args, **kwargs)\n\n if thread:\n thread.join()\n\n return back\n\n\nclass LocalAnnouncer(BroadcastServer.BroadcastServer):\n def __init__(self, server, listen_port):\n super(LocalAnnouncer, self).__init__(\"zeronet\", listen_port=listen_port)\n self.server = server\n\n self.sender_info[\"peer_id\"] = self.server.peer_id\n self.sender_info[\"port\"] = self.server.port\n self.sender_info[\"broadcast_port\"] = listen_port\n self.sender_info[\"rev\"] = config.rev\n\n self.known_peers = {}\n self.last_discover = 0\n\n def discover(self, force=False):\n self.log.debug(\"Sending discover request (force: %s)\" % force)\n self.last_discover = time.time()\n if force: # Probably new site added, clean cache\n self.known_peers = {}\n\n for peer_id, known_peer in list(self.known_peers.items()):\n if time.time() - known_peer[\"found\"] > 20 * 60:\n del(self.known_peers[peer_id])\n self.log.debug(\"Timeout, removing from known_peers: %s\" % peer_id)\n self.broadcast({\"cmd\": \"discoverRequest\", \"params\": {}}, port=self.listen_port)\n\n def actionDiscoverRequest(self, sender, params):\n back = {\n \"cmd\": \"discoverResponse\",\n \"params\": {\n \"sites_changed\": self.server.site_manager.sites_changed\n }\n }\n\n if sender[\"peer_id\"] not in self.known_peers:\n self.known_peers[sender[\"peer_id\"]] = {\"added\": time.time(), \"sites_changed\": 0, \"updated\": 0, \"found\": time.time()}\n self.log.debug(\"Got discover request from unknown peer %s (%s), time to refresh known peers\" % (sender[\"ip\"], sender[\"peer_id\"]))\n gevent.spawn_later(1.0, self.discover) # Let the response arrive first to the requester\n\n return back\n\n def actionDiscoverResponse(self, sender, params):\n if sender[\"peer_id\"] in self.known_peers:\n self.known_peers[sender[\"peer_id\"]][\"found\"] = time.time()\n if params[\"sites_changed\"] != self.known_peers.get(sender[\"peer_id\"], {}).get(\"sites_changed\"):\n # Peer's site list changed, request the list of new sites\n return {\"cmd\": \"siteListRequest\"}\n else:\n # Peer's site list is the same\n for site in self.server.sites.values():\n peer = site.peers.get(\"%s:%s\" % (sender[\"ip\"], sender[\"port\"]))\n if peer:\n peer.found(\"local\")\n\n def actionSiteListRequest(self, sender, params):\n back = []\n sites = list(self.server.sites.values())\n\n # Split adresses to group of 100 to avoid UDP size limit\n site_groups = [sites[i:i + 100] for i in range(0, len(sites), 100)]\n for site_group in site_groups:\n res = {}\n res[\"sites_changed\"] = self.server.site_manager.sites_changed\n res[\"sites\"] = [site.address_hash for site in site_group]\n back.append({\"cmd\": \"siteListResponse\", \"params\": res})\n return back\n\n def actionSiteListResponse(self, sender, params):\n s = time.time()\n peer_sites = set(params[\"sites\"])\n num_found = 0\n added_sites = []\n for site in self.server.sites.values():\n if site.address_hash in peer_sites:\n added = site.addPeer(sender[\"ip\"], sender[\"port\"], source=\"local\")\n num_found += 1\n if added:\n site.worker_manager.onPeers()\n site.updateWebsocket(peers_added=1)\n added_sites.append(site)\n\n # Save sites changed value to avoid unnecessary site list download\n if sender[\"peer_id\"] not in self.known_peers:\n self.known_peers[sender[\"peer_id\"]] = {\"added\": time.time()}\n\n self.known_peers[sender[\"peer_id\"]][\"sites_changed\"] = params[\"sites_changed\"]\n self.known_peers[sender[\"peer_id\"]][\"updated\"] = time.time()\n self.known_peers[sender[\"peer_id\"]][\"found\"] = time.time()\n\n self.log.debug(\n \"Tracker result: Discover from %s response parsed in %.3fs, found: %s added: %s of %s\" %\n (sender[\"ip\"], time.time() - s, num_found, added_sites, len(peer_sites))\n )\n\n\n@PluginManager.registerTo(\"FileServer\")\nclass FileServerPlugin(object):\n def __init__(self, *args, **kwargs):\n super(FileServerPlugin, self).__init__(*args, **kwargs)\n if config.broadcast_port and config.tor != \"always\" and not config.disable_udp:\n self.local_announcer = LocalAnnouncer(self, config.broadcast_port)\n else:\n self.local_announcer = None\n\n def start(self, *args, **kwargs):\n if self.local_announcer:\n gevent.spawn(self.local_announcer.start)\n return super(FileServerPlugin, self).start(*args, **kwargs)\n\n def stop(self):\n if self.local_announcer:\n self.local_announcer.stop()\n res = super(FileServerPlugin, self).stop()\n return res\n\n\n@PluginManager.registerTo(\"ConfigPlugin\")\nclass ConfigPlugin(object):\n def createArguments(self):\n group = self.parser.add_argument_group(\"AnnounceLocal plugin\")\n group.add_argument('--broadcast_port', help='UDP broadcasting port for local peer discovery', default=1544, type=int, metavar='port')\n\n return super(ConfigPlugin, self).createArguments()\n"
},
{
"path": "plugins/AnnounceLocal/BroadcastServer.py",
"content": "import socket\nimport logging\nimport time\nfrom contextlib import closing\n\nfrom Debug import Debug\nfrom util import UpnpPunch\nfrom util import Msgpack\n\n\nclass BroadcastServer(object):\n def __init__(self, service_name, listen_port=1544, listen_ip=''):\n self.log = logging.getLogger(\"BroadcastServer\")\n self.listen_port = listen_port\n self.listen_ip = listen_ip\n\n self.running = False\n self.sock = None\n self.sender_info = {\"service\": service_name}\n\n def createBroadcastSocket(self):\n sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)\n sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n if hasattr(socket, 'SO_REUSEPORT'):\n try:\n sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)\n except Exception as err:\n self.log.warning(\"Error setting SO_REUSEPORT: %s\" % err)\n\n binded = False\n for retry in range(3):\n try:\n sock.bind((self.listen_ip, self.listen_port))\n binded = True\n break\n except Exception as err:\n self.log.error(\n \"Socket bind to %s:%s error: %s, retry #%s\" %\n (self.listen_ip, self.listen_port, Debug.formatException(err), retry)\n )\n time.sleep(retry)\n\n if binded:\n return sock\n else:\n return False\n\n def start(self): # Listens for discover requests\n self.sock = self.createBroadcastSocket()\n if not self.sock:\n self.log.error(\"Unable to listen on port %s\" % self.listen_port)\n return\n\n self.log.debug(\"Started on port %s\" % self.listen_port)\n\n self.running = True\n\n while self.running:\n try:\n data, addr = self.sock.recvfrom(8192)\n except Exception as err:\n if self.running:\n self.log.error(\"Listener receive error: %s\" % err)\n continue\n\n if not self.running:\n break\n\n try:\n message = Msgpack.unpack(data)\n response_addr, message = self.handleMessage(addr, message)\n if message:\n self.send(response_addr, message)\n except Exception as err:\n self.log.error(\"Handlemessage error: %s\" % Debug.formatException(err))\n self.log.debug(\"Stopped listening on port %s\" % self.listen_port)\n\n def stop(self):\n self.log.debug(\"Stopping, socket: %s\" % self.sock)\n self.running = False\n if self.sock:\n self.sock.close()\n\n def send(self, addr, message):\n if type(message) is not list:\n message = [message]\n\n for message_part in message:\n message_part[\"sender\"] = self.sender_info\n\n self.log.debug(\"Send to %s: %s\" % (addr, message_part[\"cmd\"]))\n with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:\n sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n sock.sendto(Msgpack.pack(message_part), addr)\n\n def getMyIps(self):\n return UpnpPunch._get_local_ips()\n\n def broadcast(self, message, port=None):\n if not port:\n port = self.listen_port\n\n my_ips = self.getMyIps()\n addr = (\"255.255.255.255\", port)\n\n message[\"sender\"] = self.sender_info\n self.log.debug(\"Broadcast using ips %s on port %s: %s\" % (my_ips, port, message[\"cmd\"]))\n\n for my_ip in my_ips:\n try:\n with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:\n sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)\n sock.bind((my_ip, 0))\n sock.sendto(Msgpack.pack(message), addr)\n except Exception as err:\n self.log.warning(\"Error sending broadcast using ip %s: %s\" % (my_ip, err))\n\n def handleMessage(self, addr, message):\n self.log.debug(\"Got from %s: %s\" % (addr, message[\"cmd\"]))\n cmd = message[\"cmd\"]\n params = message.get(\"params\", {})\n sender = message[\"sender\"]\n sender[\"ip\"] = addr[0]\n\n func_name = \"action\" + cmd[0].upper() + cmd[1:]\n func = getattr(self, func_name, None)\n\n if sender[\"service\"] != \"zeronet\" or sender[\"peer_id\"] == self.sender_info[\"peer_id\"]:\n # Skip messages not for us or sent by us\n message = None\n elif func:\n message = func(sender, params)\n else:\n self.log.debug(\"Unknown cmd: %s\" % cmd)\n message = None\n\n return (sender[\"ip\"], sender[\"broadcast_port\"]), message\n"
},
{
"path": "plugins/AnnounceLocal/Test/TestAnnounce.py",
"content": "import time\nimport copy\n\nimport gevent\nimport pytest\nimport mock\n\nfrom AnnounceLocal import AnnounceLocalPlugin\nfrom File import FileServer\nfrom Test import Spy\n\n@pytest.fixture\ndef announcer(file_server, site):\n file_server.sites[site.address] = site\n announcer = AnnounceLocalPlugin.LocalAnnouncer(file_server, listen_port=1100)\n file_server.local_announcer = announcer\n announcer.listen_port = 1100\n announcer.sender_info[\"broadcast_port\"] = 1100\n announcer.getMyIps = mock.MagicMock(return_value=[\"127.0.0.1\"])\n announcer.discover = mock.MagicMock(return_value=False) # Don't send discover requests automatically\n gevent.spawn(announcer.start)\n time.sleep(0.5)\n\n assert file_server.local_announcer.running\n return file_server.local_announcer\n\n@pytest.fixture\ndef announcer_remote(request, site_temp):\n file_server_remote = FileServer(\"127.0.0.1\", 1545)\n file_server_remote.sites[site_temp.address] = site_temp\n announcer = AnnounceLocalPlugin.LocalAnnouncer(file_server_remote, listen_port=1101)\n file_server_remote.local_announcer = announcer\n announcer.listen_port = 1101\n announcer.sender_info[\"broadcast_port\"] = 1101\n announcer.getMyIps = mock.MagicMock(return_value=[\"127.0.0.1\"])\n announcer.discover = mock.MagicMock(return_value=False) # Don't send discover requests automatically\n gevent.spawn(announcer.start)\n time.sleep(0.5)\n\n assert file_server_remote.local_announcer.running\n\n def cleanup():\n file_server_remote.stop()\n request.addfinalizer(cleanup)\n\n\n return file_server_remote.local_announcer\n\n@pytest.mark.usefixtures(\"resetSettings\")\n@pytest.mark.usefixtures(\"resetTempSettings\")\nclass TestAnnounce:\n def testSenderInfo(self, announcer):\n sender_info = announcer.sender_info\n assert sender_info[\"port\"] > 0\n assert len(sender_info[\"peer_id\"]) == 20\n assert sender_info[\"rev\"] > 0\n\n def testIgnoreSelfMessages(self, announcer):\n # No response to messages that has same peer_id as server\n assert not announcer.handleMessage((\"0.0.0.0\", 123), {\"cmd\": \"discoverRequest\", \"sender\": announcer.sender_info, \"params\": {}})[1]\n\n # Response to messages with different peer id\n sender_info = copy.copy(announcer.sender_info)\n sender_info[\"peer_id\"] += \"-\"\n addr, res = announcer.handleMessage((\"0.0.0.0\", 123), {\"cmd\": \"discoverRequest\", \"sender\": sender_info, \"params\": {}})\n assert res[\"params\"][\"sites_changed\"] > 0\n\n def testDiscoverRequest(self, announcer, announcer_remote):\n assert len(announcer_remote.known_peers) == 0\n with Spy.Spy(announcer_remote, \"handleMessage\") as responses:\n announcer_remote.broadcast({\"cmd\": \"discoverRequest\", \"params\": {}}, port=announcer.listen_port)\n time.sleep(0.1)\n\n response_cmds = [response[1][\"cmd\"] for response in responses]\n assert response_cmds == [\"discoverResponse\", \"siteListResponse\"]\n assert len(responses[-1][1][\"params\"][\"sites\"]) == 1\n\n # It should only request siteList if sites_changed value is different from last response\n with Spy.Spy(announcer_remote, \"handleMessage\") as responses:\n announcer_remote.broadcast({\"cmd\": \"discoverRequest\", \"params\": {}}, port=announcer.listen_port)\n time.sleep(0.1)\n\n response_cmds = [response[1][\"cmd\"] for response in responses]\n assert response_cmds == [\"discoverResponse\"]\n\n def testPeerDiscover(self, announcer, announcer_remote, site):\n assert announcer.server.peer_id != announcer_remote.server.peer_id\n assert len(list(announcer.server.sites.values())[0].peers) == 0\n announcer.broadcast({\"cmd\": \"discoverRequest\"}, port=announcer_remote.listen_port)\n time.sleep(0.1)\n assert len(list(announcer.server.sites.values())[0].peers) == 1\n\n def testRecentPeerList(self, announcer, announcer_remote, site):\n assert len(site.peers_recent) == 0\n assert len(site.peers) == 0\n with Spy.Spy(announcer, \"handleMessage\") as responses:\n announcer.broadcast({\"cmd\": \"discoverRequest\", \"params\": {}}, port=announcer_remote.listen_port)\n time.sleep(0.1)\n assert [response[1][\"cmd\"] for response in responses] == [\"discoverResponse\", \"siteListResponse\"]\n assert len(site.peers_recent) == 1\n assert len(site.peers) == 1\n\n # It should update peer without siteListResponse\n last_time_found = list(site.peers.values())[0].time_found\n site.peers_recent.clear()\n with Spy.Spy(announcer, \"handleMessage\") as responses:\n announcer.broadcast({\"cmd\": \"discoverRequest\", \"params\": {}}, port=announcer_remote.listen_port)\n time.sleep(0.1)\n assert [response[1][\"cmd\"] for response in responses] == [\"discoverResponse\"]\n assert len(site.peers_recent) == 1\n assert list(site.peers.values())[0].time_found > last_time_found\n\n\n"
},
{
"path": "plugins/AnnounceLocal/Test/conftest.py",
"content": "from src.Test.conftest import *\n\nfrom Config import config\nconfig.broadcast_port = 0\n"
},
{
"path": "plugins/AnnounceLocal/Test/pytest.ini",
"content": "[pytest]\npython_files = Test*.py\naddopts = -rsxX -v --durations=6\nmarkers =\n webtest: mark a test as a webtest."
},
{
"path": "plugins/AnnounceLocal/__init__.py",
"content": "from . import AnnounceLocalPlugin"
},
{
"path": "plugins/AnnounceLocal/plugin_info.json",
"content": "{\n\t\"name\": \"AnnounceLocal\",\n\t\"description\": \"Discover LAN clients using UDP broadcasting.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/AnnounceShare/AnnounceSharePlugin.py",
"content": "import time\nimport os\nimport logging\nimport json\nimport atexit\n\nimport gevent\n\nfrom Config import config\nfrom Plugin import PluginManager\nfrom util import helper\n\n\nclass TrackerStorage(object):\n def __init__(self):\n self.log = logging.getLogger(\"TrackerStorage\")\n self.file_path = \"%s/trackers.json\" % config.data_dir\n self.load()\n self.time_discover = 0.0\n atexit.register(self.save)\n\n def getDefaultFile(self):\n return {\"shared\": {}}\n\n def onTrackerFound(self, tracker_address, type=\"shared\", my=False):\n if not tracker_address.startswith(\"zero://\"):\n return False\n\n trackers = self.getTrackers()\n added = False\n if tracker_address not in trackers:\n trackers[tracker_address] = {\n \"time_added\": time.time(),\n \"time_success\": 0,\n \"latency\": 99.0,\n \"num_error\": 0,\n \"my\": False\n }\n self.log.debug(\"New tracker found: %s\" % tracker_address)\n added = True\n\n trackers[tracker_address][\"time_found\"] = time.time()\n trackers[tracker_address][\"my\"] = my\n return added\n\n def onTrackerSuccess(self, tracker_address, latency):\n trackers = self.getTrackers()\n if tracker_address not in trackers:\n return False\n\n trackers[tracker_address][\"latency\"] = latency\n trackers[tracker_address][\"time_success\"] = time.time()\n trackers[tracker_address][\"num_error\"] = 0\n\n def onTrackerError(self, tracker_address):\n trackers = self.getTrackers()\n if tracker_address not in trackers:\n return False\n\n trackers[tracker_address][\"time_error\"] = time.time()\n trackers[tracker_address][\"num_error\"] += 1\n\n if len(self.getWorkingTrackers()) >= config.working_shared_trackers_limit:\n error_limit = 5\n else:\n error_limit = 30\n error_limit\n\n if trackers[tracker_address][\"num_error\"] > error_limit and trackers[tracker_address][\"time_success\"] < time.time() - 60 * 60:\n self.log.debug(\"Tracker %s looks down, removing.\" % tracker_address)\n del trackers[tracker_address]\n\n def getTrackers(self, type=\"shared\"):\n return self.file_content.setdefault(type, {})\n\n def getWorkingTrackers(self, type=\"shared\"):\n trackers = {\n key: tracker for key, tracker in self.getTrackers(type).items()\n if tracker[\"time_success\"] > time.time() - 60 * 60\n }\n return trackers\n\n def getFileContent(self):\n if not os.path.isfile(self.file_path):\n open(self.file_path, \"w\").write(\"{}\")\n return self.getDefaultFile()\n try:\n return json.load(open(self.file_path))\n except Exception as err:\n self.log.error(\"Error loading trackers list: %s\" % err)\n return self.getDefaultFile()\n\n def load(self):\n self.file_content = self.getFileContent()\n\n trackers = self.getTrackers()\n self.log.debug(\"Loaded %s shared trackers\" % len(trackers))\n for address, tracker in list(trackers.items()):\n tracker[\"num_error\"] = 0\n if not address.startswith(\"zero://\"):\n del trackers[address]\n\n def save(self):\n s = time.time()\n helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode(\"utf8\"))\n self.log.debug(\"Saved in %.3fs\" % (time.time() - s))\n\n def discoverTrackers(self, peers):\n if len(self.getWorkingTrackers()) > config.working_shared_trackers_limit:\n return False\n s = time.time()\n num_success = 0\n for peer in peers:\n if peer.connection and peer.connection.handshake.get(\"rev\", 0) < 3560:\n continue # Not supported\n\n res = peer.request(\"getTrackers\")\n if not res or \"error\" in res:\n continue\n\n num_success += 1\n for tracker_address in res[\"trackers\"]:\n if type(tracker_address) is bytes: # Backward compatibilitys\n tracker_address = tracker_address.decode(\"utf8\")\n added = self.onTrackerFound(tracker_address)\n if added: # Only add one tracker from one source\n break\n\n if not num_success and len(peers) < 20:\n self.time_discover = 0.0\n\n if num_success:\n self.save()\n\n self.log.debug(\"Trackers discovered from %s/%s peers in %.3fs\" % (num_success, len(peers), time.time() - s))\n\n\nif \"tracker_storage\" not in locals():\n tracker_storage = TrackerStorage()\n\n\n@PluginManager.registerTo(\"SiteAnnouncer\")\nclass SiteAnnouncerPlugin(object):\n def getTrackers(self):\n if tracker_storage.time_discover < time.time() - 5 * 60:\n tracker_storage.time_discover = time.time()\n gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers())\n trackers = super(SiteAnnouncerPlugin, self).getTrackers()\n shared_trackers = list(tracker_storage.getTrackers(\"shared\").keys())\n if shared_trackers:\n return trackers + shared_trackers\n else:\n return trackers\n\n def announceTracker(self, tracker, *args, **kwargs):\n res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs)\n if res:\n latency = res\n tracker_storage.onTrackerSuccess(tracker, latency)\n elif res is False:\n tracker_storage.onTrackerError(tracker)\n\n return res\n\n\n@PluginManager.registerTo(\"FileRequest\")\nclass FileRequestPlugin(object):\n def actionGetTrackers(self, params):\n shared_trackers = list(tracker_storage.getWorkingTrackers(\"shared\").keys())\n self.response({\"trackers\": shared_trackers})\n\n\n@PluginManager.registerTo(\"FileServer\")\nclass FileServerPlugin(object):\n def portCheck(self, *args, **kwargs):\n res = super(FileServerPlugin, self).portCheck(*args, **kwargs)\n if res and not config.tor == \"always\" and \"Bootstrapper\" in PluginManager.plugin_manager.plugin_names:\n for ip in self.ip_external_list:\n my_tracker_address = \"zero://%s:%s\" % (ip, config.fileserver_port)\n tracker_storage.onTrackerFound(my_tracker_address, my=True)\n return res\n\n\n@PluginManager.registerTo(\"ConfigPlugin\")\nclass ConfigPlugin(object):\n def createArguments(self):\n group = self.parser.add_argument_group(\"AnnounceShare plugin\")\n group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached', default=5, type=int, metavar='limit')\n\n return super(ConfigPlugin, self).createArguments()\n"
},
{
"path": "plugins/AnnounceShare/Test/TestAnnounceShare.py",
"content": "import pytest\n\nfrom AnnounceShare import AnnounceSharePlugin\nfrom Peer import Peer\nfrom Config import config\n\n\n@pytest.mark.usefixtures(\"resetSettings\")\n@pytest.mark.usefixtures(\"resetTempSettings\")\nclass TestAnnounceShare:\n def testAnnounceList(self, file_server):\n open(\"%s/trackers.json\" % config.data_dir, \"w\").write(\"{}\")\n tracker_storage = AnnounceSharePlugin.tracker_storage\n tracker_storage.load()\n peer = Peer(file_server.ip, 1544, connection_server=file_server)\n assert peer.request(\"getTrackers\")[\"trackers\"] == []\n\n tracker_storage.onTrackerFound(\"zero://%s:15441\" % file_server.ip)\n assert peer.request(\"getTrackers\")[\"trackers\"] == []\n\n # It needs to have at least one successfull announce to be shared to other peers\n tracker_storage.onTrackerSuccess(\"zero://%s:15441\" % file_server.ip, 1.0)\n assert peer.request(\"getTrackers\")[\"trackers\"] == [\"zero://%s:15441\" % file_server.ip]\n\n"
},
{
"path": "plugins/AnnounceShare/Test/conftest.py",
"content": "from src.Test.conftest import *\n\nfrom Config import config\n"
},
{
"path": "plugins/AnnounceShare/Test/pytest.ini",
"content": "[pytest]\npython_files = Test*.py\naddopts = -rsxX -v --durations=6\nmarkers =\n webtest: mark a test as a webtest."
},
{
"path": "plugins/AnnounceShare/__init__.py",
"content": "from . import AnnounceSharePlugin\n"
},
{
"path": "plugins/AnnounceShare/plugin_info.json",
"content": "{\n\t\"name\": \"AnnounceShare\",\n\t\"description\": \"Share possible trackers between clients.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/AnnounceZero/AnnounceZeroPlugin.py",
"content": "import time\nimport itertools\n\nfrom Plugin import PluginManager\nfrom util import helper\nfrom Crypt import CryptRsa\n\nallow_reload = False # No source reload supported in this plugin\ntime_full_announced = {} # Tracker address: Last announced all site to tracker\nconnection_pool = {} # Tracker address: Peer object\n\n\n# We can only import plugin host clases after the plugins are loaded\n@PluginManager.afterLoad\ndef importHostClasses():\n global Peer, AnnounceError\n from Peer import Peer\n from Site.SiteAnnouncer import AnnounceError\n\n\n# Process result got back from tracker\ndef processPeerRes(tracker_address, site, peers):\n added = 0\n\n # Onion\n found_onion = 0\n for packed_address in peers[\"onion\"]:\n found_onion += 1\n peer_onion, peer_port = helper.unpackOnionAddress(packed_address)\n if site.addPeer(peer_onion, peer_port, source=\"tracker\"):\n added += 1\n\n # Ip4\n found_ipv4 = 0\n peers_normal = itertools.chain(peers.get(\"ip4\", []), peers.get(\"ipv4\", []), peers.get(\"ipv6\", []))\n for packed_address in peers_normal:\n found_ipv4 += 1\n peer_ip, peer_port = helper.unpackAddress(packed_address)\n if site.addPeer(peer_ip, peer_port, source=\"tracker\"):\n added += 1\n\n if added:\n site.worker_manager.onPeers()\n site.updateWebsocket(peers_added=added)\n return added\n\n\n@PluginManager.registerTo(\"SiteAnnouncer\")\nclass SiteAnnouncerPlugin(object):\n def getTrackerHandler(self, protocol):\n if protocol == \"zero\":\n return self.announceTrackerZero\n else:\n return super(SiteAnnouncerPlugin, self).getTrackerHandler(protocol)\n\n def announceTrackerZero(self, tracker_address, mode=\"start\", num_want=10):\n global time_full_announced\n s = time.time()\n\n need_types = [\"ip4\"] # ip4 for backward compatibility reasons\n need_types += self.site.connection_server.supported_ip_types\n if self.site.connection_server.tor_manager.enabled:\n need_types.append(\"onion\")\n\n if mode == \"start\" or mode == \"more\": # Single: Announce only this site\n sites = [self.site]\n full_announce = False\n else: # Multi: Announce all currently serving site\n full_announce = True\n if time.time() - time_full_announced.get(tracker_address, 0) < 60 * 15: # No reannounce all sites within short time\n return None\n time_full_announced[tracker_address] = time.time()\n from Site import SiteManager\n sites = [site for site in SiteManager.site_manager.sites.values() if site.isServing()]\n\n # Create request\n add_types = self.getOpenedServiceTypes()\n request = {\n \"hashes\": [], \"onions\": [], \"port\": self.fileserver_port, \"need_types\": need_types, \"need_num\": 20, \"add\": add_types\n }\n for site in sites:\n if \"onion\" in add_types:\n onion = self.site.connection_server.tor_manager.getOnion(site.address)\n request[\"onions\"].append(onion)\n request[\"hashes\"].append(site.address_hash)\n\n # Tracker can remove sites that we don't announce\n if full_announce:\n request[\"delete\"] = True\n\n # Sent request to tracker\n tracker_peer = connection_pool.get(tracker_address) # Re-use tracker connection if possible\n if not tracker_peer:\n tracker_ip, tracker_port = tracker_address.rsplit(\":\", 1)\n tracker_peer = Peer(str(tracker_ip), int(tracker_port), connection_server=self.site.connection_server)\n tracker_peer.is_tracker_connection = True\n connection_pool[tracker_address] = tracker_peer\n\n res = tracker_peer.request(\"announce\", request)\n\n if not res or \"peers\" not in res:\n if full_announce:\n time_full_announced[tracker_address] = 0\n raise AnnounceError(\"Invalid response: %s\" % res)\n\n # Add peers from response to site\n site_index = 0\n peers_added = 0\n for site_res in res[\"peers\"]:\n site = sites[site_index]\n peers_added += processPeerRes(tracker_address, site, site_res)\n site_index += 1\n\n # Check if we need to sign prove the onion addresses\n if \"onion_sign_this\" in res:\n self.site.log.debug(\"Signing %s for %s to add %s onions\" % (res[\"onion_sign_this\"], tracker_address, len(sites)))\n request[\"onion_signs\"] = {}\n request[\"onion_sign_this\"] = res[\"onion_sign_this\"]\n request[\"need_num\"] = 0\n for site in sites:\n onion = self.site.connection_server.tor_manager.getOnion(site.address)\n publickey = self.site.connection_server.tor_manager.getPublickey(onion)\n if publickey not in request[\"onion_signs\"]:\n sign = CryptRsa.sign(res[\"onion_sign_this\"].encode(\"utf8\"), self.site.connection_server.tor_manager.getPrivatekey(onion))\n request[\"onion_signs\"][publickey] = sign\n res = tracker_peer.request(\"announce\", request)\n if not res or \"onion_sign_this\" in res:\n if full_announce:\n time_full_announced[tracker_address] = 0\n raise AnnounceError(\"Announce onion address to failed: %s\" % res)\n\n if full_announce:\n tracker_peer.remove() # Close connection, we don't need it in next 5 minute\n\n self.site.log.debug(\n \"Tracker announce result: zero://%s (sites: %s, new peers: %s, add: %s, mode: %s) in %.3fs\" %\n (tracker_address, site_index, peers_added, add_types, mode, time.time() - s)\n )\n\n return True\n"
},
{
"path": "plugins/AnnounceZero/__init__.py",
"content": "from . import AnnounceZeroPlugin"
},
{
"path": "plugins/AnnounceZero/plugin_info.json",
"content": "{\n\t\"name\": \"AnnounceZero\",\n\t\"description\": \"Announce using ZeroNet protocol.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/Benchmark/BenchmarkDb.py",
"content": "import os\nimport json\nimport contextlib\nimport time\n\nfrom Plugin import PluginManager\nfrom Config import config\n\n\n@PluginManager.registerTo(\"Actions\")\nclass ActionsPlugin:\n def getBenchmarkTests(self, online=False):\n tests = super().getBenchmarkTests(online)\n tests.extend([\n {\"func\": self.testDbConnect, \"num\": 10, \"time_standard\": 0.27},\n {\"func\": self.testDbInsert, \"num\": 10, \"time_standard\": 0.91},\n {\"func\": self.testDbInsertMultiuser, \"num\": 1, \"time_standard\": 0.57},\n {\"func\": self.testDbQueryIndexed, \"num\": 1000, \"time_standard\": 0.84},\n {\"func\": self.testDbQueryNotIndexed, \"num\": 1000, \"time_standard\": 1.30}\n ])\n return tests\n\n\n @contextlib.contextmanager\n def getTestDb(self):\n from Db import Db\n path = \"%s/benchmark.db\" % config.data_dir\n if os.path.isfile(path):\n os.unlink(path)\n schema = {\n \"db_name\": \"TestDb\",\n \"db_file\": path,\n \"maps\": {\n \".*\": {\n \"to_table\": {\n \"test\": \"test\"\n }\n }\n },\n \"tables\": {\n \"test\": {\n \"cols\": [\n [\"test_id\", \"INTEGER\"],\n [\"title\", \"TEXT\"],\n [\"json_id\", \"INTEGER REFERENCES json (json_id)\"]\n ],\n \"indexes\": [\"CREATE UNIQUE INDEX test_key ON test(test_id, json_id)\"],\n \"schema_changed\": 1426195822\n }\n }\n }\n\n db = Db.Db(schema, path)\n\n yield db\n\n db.close()\n if os.path.isfile(path):\n os.unlink(path)\n\n def testDbConnect(self, num_run=1):\n import sqlite3\n for i in range(num_run):\n with self.getTestDb() as db:\n db.checkTables()\n yield \".\"\n yield \"(SQLite version: %s, API: %s)\" % (sqlite3.sqlite_version, sqlite3.version)\n\n def testDbInsert(self, num_run=1):\n yield \"x 1000 lines \"\n for u in range(num_run):\n with self.getTestDb() as db:\n db.checkTables()\n data = {\"test\": []}\n for i in range(1000): # 1000 line of data\n data[\"test\"].append({\"test_id\": i, \"title\": \"Testdata for %s message %s\" % (u, i)})\n json.dump(data, open(\"%s/test_%s.json\" % (config.data_dir, u), \"w\"))\n db.updateJson(\"%s/test_%s.json\" % (config.data_dir, u))\n os.unlink(\"%s/test_%s.json\" % (config.data_dir, u))\n assert db.execute(\"SELECT COUNT(*) FROM test\").fetchone()[0] == 1000\n yield \".\"\n\n def fillTestDb(self, db):\n db.checkTables()\n cur = db.getCursor()\n cur.logging = False\n for u in range(100, 200): # 100 user\n data = {\"test\": []}\n for i in range(100): # 1000 line of data\n data[\"test\"].append({\"test_id\": i, \"title\": \"Testdata for %s message %s\" % (u, i)})\n json.dump(data, open(\"%s/test_%s.json\" % (config.data_dir, u), \"w\"))\n db.updateJson(\"%s/test_%s.json\" % (config.data_dir, u), cur=cur)\n os.unlink(\"%s/test_%s.json\" % (config.data_dir, u))\n if u % 10 == 0:\n yield \".\"\n\n def testDbInsertMultiuser(self, num_run=1):\n yield \"x 100 users x 100 lines \"\n for u in range(num_run):\n with self.getTestDb() as db:\n for progress in self.fillTestDb(db):\n yield progress\n num_rows = db.execute(\"SELECT COUNT(*) FROM test\").fetchone()[0]\n assert num_rows == 10000, \"%s != 10000\" % num_rows\n\n def testDbQueryIndexed(self, num_run=1):\n s = time.time()\n with self.getTestDb() as db:\n for progress in self.fillTestDb(db):\n pass\n yield \" (Db warmup done in %.3fs) \" % (time.time() - s)\n found_total = 0\n for i in range(num_run): # 1000x by test_id\n found = 0\n res = db.execute(\"SELECT * FROM test WHERE test_id = %s\" % (i % 100))\n for row in res:\n found_total += 1\n found += 1\n del(res)\n yield \".\"\n assert found == 100, \"%s != 100 (i: %s)\" % (found, i)\n yield \"Found: %s\" % found_total\n\n def testDbQueryNotIndexed(self, num_run=1):\n s = time.time()\n with self.getTestDb() as db:\n for progress in self.fillTestDb(db):\n pass\n yield \" (Db warmup done in %.3fs) \" % (time.time() - s)\n found_total = 0\n for i in range(num_run): # 1000x by test_id\n found = 0\n res = db.execute(\"SELECT * FROM test WHERE json_id = %s\" % i)\n for row in res:\n found_total += 1\n found += 1\n yield \".\"\n del(res)\n if i == 0 or i > 100:\n assert found == 0, \"%s != 0 (i: %s)\" % (found, i)\n else:\n assert found == 100, \"%s != 100 (i: %s)\" % (found, i)\n yield \"Found: %s\" % found_total\n"
},
{
"path": "plugins/Benchmark/BenchmarkPack.py",
"content": "import os\nimport io\nfrom collections import OrderedDict\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom util import Msgpack\n\n\n@PluginManager.registerTo(\"Actions\")\nclass ActionsPlugin:\n def createZipFile(self, path):\n import zipfile\n test_data = b\"Test\" * 1024\n file_name = b\"\\xc3\\x81rv\\xc3\\xadzt\\xc5\\xb1r\\xc5\\x91%s.txt\".decode(\"utf8\")\n with zipfile.ZipFile(path, 'w') as archive:\n for y in range(100):\n zip_info = zipfile.ZipInfo(file_name % y, (1980, 1, 1, 0, 0, 0))\n zip_info.compress_type = zipfile.ZIP_DEFLATED\n zip_info.create_system = 3\n zip_info.flag_bits = 0\n zip_info.external_attr = 25165824\n archive.writestr(zip_info, test_data)\n\n def testPackZip(self, num_run=1):\n \"\"\"\n Test zip file creating\n \"\"\"\n yield \"x 100 x 5KB \"\n from Crypt import CryptHash\n zip_path = '%s/test.zip' % config.data_dir\n for i in range(num_run):\n self.createZipFile(zip_path)\n yield \".\"\n\n archive_size = os.path.getsize(zip_path) / 1024\n yield \"(Generated file size: %.2fkB)\" % archive_size\n\n hash = CryptHash.sha512sum(open(zip_path, \"rb\"))\n valid = \"cb32fb43783a1c06a2170a6bc5bb228a032b67ff7a1fd7a5efb9b467b400f553\"\n assert hash == valid, \"Invalid hash: %s != %s\" % (hash, valid)\n os.unlink(zip_path)\n\n def testUnpackZip(self, num_run=1):\n \"\"\"\n Test zip file reading\n \"\"\"\n yield \"x 100 x 5KB \"\n import zipfile\n zip_path = '%s/test.zip' % config.data_dir\n test_data = b\"Test\" * 1024\n file_name = b\"\\xc3\\x81rv\\xc3\\xadzt\\xc5\\xb1r\\xc5\\x91\".decode(\"utf8\")\n\n self.createZipFile(zip_path)\n for i in range(num_run):\n with zipfile.ZipFile(zip_path) as archive:\n for f in archive.filelist:\n assert f.filename.startswith(file_name), \"Invalid filename: %s != %s\" % (f.filename, file_name)\n data = archive.open(f.filename).read()\n assert archive.open(f.filename).read() == test_data, \"Invalid data: %s...\" % data[0:30]\n yield \".\"\n\n os.unlink(zip_path)\n\n def createArchiveFile(self, path, archive_type=\"gz\"):\n import tarfile\n import gzip\n\n # Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime\n def nodate_write_gzip_header(self):\n self._write_mtime = 0\n original_write_gzip_header(self)\n\n test_data_io = io.BytesIO(b\"Test\" * 1024)\n file_name = b\"\\xc3\\x81rv\\xc3\\xadzt\\xc5\\xb1r\\xc5\\x91%s.txt\".decode(\"utf8\")\n\n original_write_gzip_header = gzip.GzipFile._write_gzip_header\n gzip.GzipFile._write_gzip_header = nodate_write_gzip_header\n with tarfile.open(path, 'w:%s' % archive_type) as archive:\n for y in range(100):\n test_data_io.seek(0)\n tar_info = tarfile.TarInfo(file_name % y)\n tar_info.size = 4 * 1024\n archive.addfile(tar_info, test_data_io)\n\n def testPackArchive(self, num_run=1, archive_type=\"gz\"):\n \"\"\"\n Test creating tar archive files\n \"\"\"\n yield \"x 100 x 5KB \"\n from Crypt import CryptHash\n\n hash_valid_db = {\n \"gz\": \"92caec5121a31709cbbc8c11b0939758e670b055bbbe84f9beb3e781dfde710f\",\n \"bz2\": \"b613f41e6ee947c8b9b589d3e8fa66f3e28f63be23f4faf015e2f01b5c0b032d\",\n \"xz\": \"ae43892581d770959c8d993daffab25fd74490b7cf9fafc7aaee746f69895bcb\",\n }\n archive_path = '%s/test.tar.%s' % (config.data_dir, archive_type)\n for i in range(num_run):\n self.createArchiveFile(archive_path, archive_type=archive_type)\n yield \".\"\n\n archive_size = os.path.getsize(archive_path) / 1024\n yield \"(Generated file size: %.2fkB)\" % archive_size\n\n hash = CryptHash.sha512sum(open(\"%s/test.tar.%s\" % (config.data_dir, archive_type), \"rb\"))\n valid = hash_valid_db[archive_type]\n assert hash == valid, \"Invalid hash: %s != %s
\" % (hash, valid)\n\n if os.path.isfile(archive_path):\n os.unlink(archive_path)\n\n def testUnpackArchive(self, num_run=1, archive_type=\"gz\"):\n \"\"\"\n Test reading tar archive files\n \"\"\"\n yield \"x 100 x 5KB \"\n import tarfile\n\n test_data = b\"Test\" * 1024\n file_name = b\"\\xc3\\x81rv\\xc3\\xadzt\\xc5\\xb1r\\xc5\\x91%s.txt\".decode(\"utf8\")\n archive_path = '%s/test.tar.%s' % (config.data_dir, archive_type)\n self.createArchiveFile(archive_path, archive_type=archive_type)\n for i in range(num_run):\n with tarfile.open(archive_path, 'r:%s' % archive_type) as archive:\n for y in range(100):\n assert archive.extractfile(file_name % y).read() == test_data\n yield \".\"\n if os.path.isfile(archive_path):\n os.unlink(archive_path)\n\n def testPackMsgpack(self, num_run=1):\n \"\"\"\n Test msgpack encoding\n \"\"\"\n yield \"x 100 x 5KB \"\n binary = b'fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv'\n data = OrderedDict(\n sorted({\"int\": 1024 * 1024 * 1024, \"float\": 12345.67890, \"text\": \"hello\" * 1024, \"binary\": binary}.items())\n )\n data_packed_valid = b'\\x84\\xa6binary\\xc5\\x01\\x00fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv\\xa5float\\xcb@\\xc8\\x1c\\xd6\\xe61\\xf8\\xa1\\xa3int\\xce@\\x00\\x00\\x00\\xa4text\\xda\\x14\\x00'\n data_packed_valid += b'hello' * 1024\n for y in range(num_run):\n for i in range(100):\n data_packed = Msgpack.pack(data)\n yield \".\"\n assert data_packed == data_packed_valid, \"%s
!=
%s\" % (repr(data_packed), repr(data_packed_valid))\n\n def testUnpackMsgpack(self, num_run=1):\n \"\"\"\n Test msgpack decoding\n \"\"\"\n yield \"x 5KB \"\n binary = b'fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv'\n data = OrderedDict(\n sorted({\"int\": 1024 * 1024 * 1024, \"float\": 12345.67890, \"text\": \"hello\" * 1024, \"binary\": binary}.items())\n )\n data_packed = b'\\x84\\xa6binary\\xc5\\x01\\x00fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv\\xa5float\\xcb@\\xc8\\x1c\\xd6\\xe61\\xf8\\xa1\\xa3int\\xce@\\x00\\x00\\x00\\xa4text\\xda\\x14\\x00'\n data_packed += b'hello' * 1024\n for y in range(num_run):\n data_unpacked = Msgpack.unpack(data_packed, decode=False)\n yield \".\"\n assert data_unpacked == data, \"%s
!=
%s\" % (data_unpacked, data)\n\n def testUnpackMsgpackStreaming(self, num_run=1, fallback=False):\n \"\"\"\n Test streaming msgpack decoding\n \"\"\"\n yield \"x 1000 x 5KB \"\n binary = b'fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv'\n data = OrderedDict(\n sorted({\"int\": 1024 * 1024 * 1024, \"float\": 12345.67890, \"text\": \"hello\" * 1024, \"binary\": binary}.items())\n )\n data_packed = b'\\x84\\xa6binary\\xc5\\x01\\x00fqv\\xf0\\x1a\"e\\x10,\\xbe\\x9cT\\x9e(\\xa5]u\\x072C\\x8c\\x15\\xa2\\xa8\\x93Sw)\\x19\\x02\\xdd\\t\\xfb\\xf67\\x88\\xd9\\xee\\x86\\xa1\\xe4\\xb6,\\xc6\\x14\\xbb\\xd7$z\\x1d\\xb2\\xda\\x85\\xf5\\xa0\\x97^\\x01*\\xaf\\xd3\\xb0!\\xb7\\x9d\\xea\\x89\\xbbh8\\xa1\"\\xa7]e(@\\xa2\\xa5g\\xb7[\\xae\\x8eE\\xc2\\x9fL\\xb6s\\x19\\x19\\r\\xc8\\x04S\\xd0N\\xe4]?/\\x01\\xea\\xf6\\xec\\xd1\\xb3\\xc2\\x91\\x86\\xd7\\xf4K\\xdf\\xc2lV\\xf4\\xe8\\x80\\xfc\\x8ep\\xbb\\x82\\xb3\\x86\\x98F\\x1c\\xecS\\xc8\\x15\\xcf\\xdc\\xf1\\xed\\xfc\\xd8\\x18r\\xf9\\x80\\x0f\\xfa\\x8cO\\x97(\\x0b]\\xf1\\xdd\\r\\xe7\\xbf\\xed\\x06\\xbd\\x1b?\\xc5\\xa0\\xd7a\\x82\\xf3\\xa8\\xe6@\\xf3\\ri\\xa1\\xb10\\xf6\\xd4W\\xbc\\x86\\x1a\\xbb\\xfd\\x94!bS\\xdb\\xaeM\\x92\\x00#\\x0b\\xf7\\xad\\xe9\\xc2\\x8e\\x86\\xbfi![%\\xd31]\\xc6\\xfc2\\xc9\\xda\\xc6v\\x82P\\xcc\\xa9\\xea\\xb9\\xff\\xf6\\xc8\\x17iD\\xcf\\xf3\\xeeI\\x04\\xe9\\xa1\\x19\\xbb\\x01\\x92\\xf5nn4K\\xf8\\xbb\\xc6\\x17e>\\xa7 \\xbbv\\xa5float\\xcb@\\xc8\\x1c\\xd6\\xe61\\xf8\\xa1\\xa3int\\xce@\\x00\\x00\\x00\\xa4text\\xda\\x14\\x00'\n data_packed += b'hello' * 1024\n for i in range(num_run):\n unpacker = Msgpack.getUnpacker(decode=False, fallback=fallback)\n for y in range(1000):\n unpacker.feed(data_packed)\n for data_unpacked in unpacker:\n pass\n yield \".\"\n assert data == data_unpacked, \"%s != %s\" % (data_unpacked, data)\n" }, { "path": "plugins/Benchmark/BenchmarkPlugin.py", "content": "import os\nimport time\nimport io\nimport math\nimport hashlib\nimport re\nimport sys\n\nfrom Config import config\nfrom Crypt import CryptHash\nfrom Plugin import PluginManager\nfrom Debug import Debug\nfrom util import helper\n\nplugin_dir = os.path.dirname(__file__)\n\nbenchmark_key = None\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n @helper.encodeResponse\n def actionBenchmark(self):\n global benchmark_key\n script_nonce = self.getScriptNonce()\n if not benchmark_key:\n benchmark_key = CryptHash.random(encoding=\"base64\")\n self.sendHeader(script_nonce=script_nonce)\n\n if \"Multiuser\" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:\n yield \"This function is disabled on this proxy\"\n return\n\n data = self.render(\n plugin_dir + \"/media/benchmark.html\",\n script_nonce=script_nonce,\n benchmark_key=benchmark_key,\n filter=re.sub(\"[^A-Za-z0-9]\", \"\", self.get.get(\"filter\", \"\"))\n )\n yield data\n\n @helper.encodeResponse\n def actionBenchmarkResult(self):\n global benchmark_key\n if self.get.get(\"benchmark_key\", \"\") != benchmark_key:\n return self.error403(\"Invalid benchmark key\")\n\n self.sendHeader(content_type=\"text/plain\", noscript=True)\n\n if \"Multiuser\" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:\n yield \"This function is disabled on this proxy\"\n return\n\n yield \" \" * 1024 # Head (required for streaming)\n\n import main\n s = time.time()\n\n for part in main.actions.testBenchmark(filter=self.get.get(\"filter\", \"\")):\n yield part\n\n yield \"\\n - Total time: %.3fs\" % (time.time() - s)\n\n\n@PluginManager.registerTo(\"Actions\")\nclass ActionsPlugin:\n def getMultiplerTitle(self, multipler):\n if multipler < 0.3:\n multipler_title = \"Sloooow\"\n elif multipler < 0.6:\n multipler_title = \"Ehh\"\n elif multipler < 0.8:\n multipler_title = \"Goodish\"\n elif multipler < 1.2:\n multipler_title = \"OK\"\n elif multipler < 1.7:\n multipler_title = \"Fine\"\n elif multipler < 2.5:\n multipler_title = \"Fast\"\n elif multipler < 3.5:\n multipler_title = \"WOW\"\n else:\n multipler_title = \"Insane!!\"\n return multipler_title\n\n def formatResult(self, taken, standard):\n if not standard:\n return \" Done in %.3fs\" % taken\n\n if taken > 0:\n multipler = standard / taken\n else:\n multipler = 99\n multipler_title = self.getMultiplerTitle(multipler)\n\n return \" Done in %.3fs = %s (%.2fx)\" % (taken, multipler_title, multipler)\n\n def getBenchmarkTests(self, online=False):\n if hasattr(super(), \"getBenchmarkTests\"):\n tests = super().getBenchmarkTests(online)\n else:\n tests = []\n\n tests.extend([\n {\"func\": self.testHdPrivatekey, \"num\": 50, \"time_standard\": 0.57},\n {\"func\": self.testSign, \"num\": 20, \"time_standard\": 0.46},\n {\"func\": self.testVerify, \"kwargs\": {\"lib_verify\": \"sslcrypto_fallback\"}, \"num\": 20, \"time_standard\": 0.38},\n {\"func\": self.testVerify, \"kwargs\": {\"lib_verify\": \"sslcrypto\"}, \"num\": 200, \"time_standard\": 0.30},\n {\"func\": self.testVerify, \"kwargs\": {\"lib_verify\": \"libsecp256k1\"}, \"num\": 200, \"time_standard\": 0.10},\n\n {\"func\": self.testPackMsgpack, \"num\": 100, \"time_standard\": 0.35},\n {\"func\": self.testUnpackMsgpackStreaming, \"kwargs\": {\"fallback\": False}, \"num\": 100, \"time_standard\": 0.35},\n {\"func\": self.testUnpackMsgpackStreaming, \"kwargs\": {\"fallback\": True}, \"num\": 10, \"time_standard\": 0.5},\n\n {\"func\": self.testPackZip, \"num\": 5, \"time_standard\": 0.065},\n {\"func\": self.testPackArchive, \"kwargs\": {\"archive_type\": \"gz\"}, \"num\": 5, \"time_standard\": 0.08},\n {\"func\": self.testPackArchive, \"kwargs\": {\"archive_type\": \"bz2\"}, \"num\": 5, \"time_standard\": 0.68},\n {\"func\": self.testPackArchive, \"kwargs\": {\"archive_type\": \"xz\"}, \"num\": 5, \"time_standard\": 0.47},\n {\"func\": self.testUnpackZip, \"num\": 20, \"time_standard\": 0.25},\n {\"func\": self.testUnpackArchive, \"kwargs\": {\"archive_type\": \"gz\"}, \"num\": 20, \"time_standard\": 0.28},\n {\"func\": self.testUnpackArchive, \"kwargs\": {\"archive_type\": \"bz2\"}, \"num\": 20, \"time_standard\": 0.83},\n {\"func\": self.testUnpackArchive, \"kwargs\": {\"archive_type\": \"xz\"}, \"num\": 20, \"time_standard\": 0.38},\n\n {\"func\": self.testCryptHash, \"kwargs\": {\"hash_type\": \"sha256\"}, \"num\": 10, \"time_standard\": 0.50},\n {\"func\": self.testCryptHash, \"kwargs\": {\"hash_type\": \"sha512\"}, \"num\": 10, \"time_standard\": 0.33},\n {\"func\": self.testCryptHashlib, \"kwargs\": {\"hash_type\": \"sha3_256\"}, \"num\": 10, \"time_standard\": 0.33},\n {\"func\": self.testCryptHashlib, \"kwargs\": {\"hash_type\": \"sha3_512\"}, \"num\": 10, \"time_standard\": 0.65},\n\n {\"func\": self.testRandom, \"num\": 100, \"time_standard\": 0.08},\n ])\n\n if online:\n tests += [\n {\"func\": self.testHttps, \"num\": 1, \"time_standard\": 2.1}\n ]\n return tests\n\n def testBenchmark(self, num_multipler=1, online=False, num_run=None, filter=None):\n \"\"\"\n Run benchmark on client functions\n \"\"\"\n tests = self.getBenchmarkTests(online=online)\n\n if filter:\n tests = [test for test in tests[:] if filter.lower() in test[\"func\"].__name__.lower()]\n\n yield \"\\n\"\n res = {}\n res_time_taken = {}\n multiplers = []\n for test in tests:\n s = time.time()\n if num_run:\n num_run_test = num_run\n else:\n num_run_test = math.ceil(test[\"num\"] * num_multipler)\n func = test[\"func\"]\n func_name = func.__name__\n kwargs = test.get(\"kwargs\", {})\n key = \"%s %s\" % (func_name, kwargs)\n if kwargs:\n yield \"* Running %s (%s) x %s \" % (func_name, kwargs, num_run_test)\n else:\n yield \"* Running %s x %s \" % (func_name, num_run_test)\n i = 0\n try:\n for progress in func(num_run_test, **kwargs):\n i += 1\n if num_run_test > 10:\n should_print = i % (num_run_test / 10) == 0 or progress != \".\"\n else:\n should_print = True\n\n if should_print:\n if num_run_test == 1 and progress == \".\":\n progress = \"...\"\n yield progress\n time_taken = time.time() - s\n if num_run:\n time_standard = 0\n else:\n time_standard = test[\"time_standard\"] * num_multipler\n yield self.formatResult(time_taken, time_standard)\n yield \"\\n\"\n res[key] = \"ok\"\n res_time_taken[key] = time_taken\n multiplers.append(time_standard / max(time_taken, 0.001))\n except Exception as err:\n res[key] = err\n yield \"Failed!\\n! Error: %s\\n\\n\" % Debug.formatException(err)\n\n yield \"\\n== Result ==\\n\"\n\n # Check verification speed\n if \"testVerify {'lib_verify': 'sslcrypto'}\" in res_time_taken:\n speed_order = [\"sslcrypto_fallback\", \"sslcrypto\", \"libsecp256k1\"]\n time_taken = {}\n for lib_verify in speed_order:\n time_taken[lib_verify] = res_time_taken[\"testVerify {'lib_verify': '%s'}\" % lib_verify]\n\n time_taken[\"sslcrypto_fallback\"] *= 10 # fallback benchmark only run 20 times instead of 200\n speedup_sslcrypto = time_taken[\"sslcrypto_fallback\"] / time_taken[\"sslcrypto\"]\n speedup_libsecp256k1 = time_taken[\"sslcrypto_fallback\"] / time_taken[\"libsecp256k1\"]\n\n yield \"\\n* Verification speedup:\\n\"\n yield \" - OpenSSL: %.1fx (reference: 7.0x)\\n\" % speedup_sslcrypto\n yield \" - libsecp256k1: %.1fx (reference: 23.8x)\\n\" % speedup_libsecp256k1\n\n if speedup_sslcrypto < 2:\n res[\"Verification speed\"] = \"error: OpenSSL speedup low: %.1fx\" % speedup_sslcrypto\n\n if speedup_libsecp256k1 < speedup_sslcrypto:\n res[\"Verification speed\"] = \"error: libsecp256k1 speedup low: %.1fx\" % speedup_libsecp256k1\n\n if not res:\n yield \"! No tests found\"\n if config.action == \"test\":\n sys.exit(1)\n else:\n num_failed = len([res_key for res_key, res_val in res.items() if res_val != \"ok\"])\n num_success = len([res_key for res_key, res_val in res.items() if res_val == \"ok\"])\n yield \"\\n* Tests:\\n\"\n yield \" - Total: %s tests\\n\" % len(res)\n yield \" - Success: %s tests\\n\" % num_success\n yield \" - Failed: %s tests\\n\" % num_failed\n if any(multiplers):\n multipler_avg = sum(multiplers) / len(multiplers)\n multipler_title = self.getMultiplerTitle(multipler_avg)\n yield \" - Average speed factor: %.2fx (%s)\\n\" % (multipler_avg, multipler_title)\n\n # Display errors\n for res_key, res_val in res.items():\n if res_val != \"ok\":\n yield \" ! %s %s\\n\" % (res_key, res_val)\n\n if num_failed != 0 and config.action == \"test\":\n sys.exit(1)\n\n def testHttps(self, num_run=1):\n \"\"\"\n Test https connection with valid and invalid certs\n \"\"\"\n import urllib.request\n import urllib.error\n\n body = urllib.request.urlopen(\"https://google.com\").read()\n assert len(body) > 100\n yield \".\"\n\n badssl_urls = [\n \"https://expired.badssl.com/\",\n \"https://wrong.host.badssl.com/\",\n \"https://self-signed.badssl.com/\",\n \"https://untrusted-root.badssl.com/\"\n ]\n for badssl_url in badssl_urls:\n try:\n body = urllib.request.urlopen(badssl_url).read()\n https_err = None\n except urllib.error.URLError as err:\n https_err = err\n assert https_err\n yield \".\"\n\n def testCryptHash(self, num_run=1, hash_type=\"sha256\"):\n \"\"\"\n Test hashing functions\n \"\"\"\n yield \"(5MB) \"\n\n from Crypt import CryptHash\n\n hash_types = {\n \"sha256\": {\"func\": CryptHash.sha256sum, \"hash_valid\": \"8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa\"},\n \"sha512\": {\"func\": CryptHash.sha512sum, \"hash_valid\": \"9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d\"}\n }\n hash_func = hash_types[hash_type][\"func\"]\n hash_valid = hash_types[hash_type][\"hash_valid\"]\n\n data = io.BytesIO(b\"Hello\" * 1024 * 1024) # 5MB\n for i in range(num_run):\n data.seek(0)\n hash = hash_func(data)\n yield \".\"\n assert hash == hash_valid, \"%s != %s\" % (hash, hash_valid)\n\n def testCryptHashlib(self, num_run=1, hash_type=\"sha3_256\"):\n \"\"\"\n Test SHA3 hashing functions\n \"\"\"\n yield \"x 5MB \"\n\n hash_types = {\n \"sha3_256\": {\"func\": hashlib.sha3_256, \"hash_valid\": \"c8aeb3ef9fe5d6404871c0d2a4410a4d4e23268e06735648c9596f436c495f7e\"},\n \"sha3_512\": {\"func\": hashlib.sha3_512, \"hash_valid\": \"b75dba9472d8af3cc945ce49073f3f8214d7ac12086c0453fb08944823dee1ae83b3ffbc87a53a57cc454521d6a26fe73ff0f3be38dddf3f7de5d7692ebc7f95\"},\n }\n\n hash_func = hash_types[hash_type][\"func\"]\n hash_valid = hash_types[hash_type][\"hash_valid\"]\n\n data = io.BytesIO(b\"Hello\" * 1024 * 1024) # 5MB\n for i in range(num_run):\n data.seek(0)\n h = hash_func()\n while 1:\n buff = data.read(1024 * 64)\n if not buff:\n break\n h.update(buff)\n hash = h.hexdigest()\n yield \".\"\n assert hash == hash_valid, \"%s != %s\" % (hash, hash_valid)\n\n def testRandom(self, num_run=1):\n \"\"\"\n Test generating random data\n \"\"\"\n yield \"x 1000 x 256 bytes \"\n for i in range(num_run):\n data_last = None\n for y in range(1000):\n data = os.urandom(256)\n assert data != data_last\n assert len(data) == 256\n data_last = data\n yield \".\"\n\n def testHdPrivatekey(self, num_run=2):\n \"\"\"\n Test generating deterministic private keys from a master seed\n \"\"\"\n from Crypt import CryptBitcoin\n seed = \"e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38\"\n privatekeys = []\n for i in range(num_run):\n privatekeys.append(CryptBitcoin.hdPrivatekey(seed, i * 10))\n yield \".\"\n valid = \"5JSbeF5PevdrsYjunqpg7kAGbnCVYa1T4APSL3QRu8EoAmXRc7Y\"\n assert privatekeys[0] == valid, \"%s != %s\" % (privatekeys[0], valid)\n if len(privatekeys) > 1:\n assert privatekeys[0] != privatekeys[-1]\n\n def testSign(self, num_run=1):\n \"\"\"\n Test signing data using a private key\n \"\"\"\n from Crypt import CryptBitcoin\n data = \"Hello\" * 1024\n privatekey = \"5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk\"\n for i in range(num_run):\n yield \".\"\n sign = CryptBitcoin.sign(data, privatekey)\n valid = \"G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w=\"\n assert sign == valid, \"%s != %s\" % (sign, valid)\n\n def testVerify(self, num_run=1, lib_verify=\"sslcrypto\"):\n \"\"\"\n Test verification of generated signatures\n \"\"\"\n from Crypt import CryptBitcoin\n CryptBitcoin.loadLib(lib_verify, silent=True)\n\n\n data = \"Hello\" * 1024\n privatekey = \"5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk\"\n address = CryptBitcoin.privatekeyToAddress(privatekey)\n sign = \"G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w=\"\n\n for i in range(num_run):\n ok = CryptBitcoin.verify(data, address, sign, lib_verify=lib_verify)\n yield \".\"\n assert ok, \"does not verify from %s\" % address\n\n if lib_verify == \"sslcrypto\":\n yield(\"(%s)\" % CryptBitcoin.sslcrypto.ecc.get_backend())\n\n def testPortCheckers(self):\n \"\"\"\n Test all active open port checker\n \"\"\"\n from Peer import PeerPortchecker\n for ip_type, func_names in PeerPortchecker.PeerPortchecker.checker_functions.items():\n yield \"\\n- %s:\" % ip_type\n for func_name in func_names:\n yield \"\\n - Tracker %s: \" % func_name\n try:\n for res in self.testPortChecker(func_name):\n yield res\n except Exception as err:\n yield Debug.formatException(err)\n\n def testPortChecker(self, func_name):\n \"\"\"\n Test single open port checker\n \"\"\"\n from Peer import PeerPortchecker\n peer_portchecker = PeerPortchecker.PeerPortchecker(None)\n announce_func = getattr(peer_portchecker, func_name)\n res = announce_func(3894)\n yield res\n\n def testAll(self):\n \"\"\"\n Run all tests to check system compatibility with ZeroNet functions\n \"\"\"\n for progress in self.testBenchmark(online=not config.offline, num_run=1):\n yield progress\n\n\n@PluginManager.registerTo(\"ConfigPlugin\")\nclass ConfigPlugin(object):\n def createArguments(self):\n back = super(ConfigPlugin, self).createArguments()\n if self.getCmdlineValue(\"test\") == \"benchmark\":\n self.test_parser.add_argument(\n '--num_multipler', help='Benchmark run time multipler',\n default=1.0, type=float, metavar='num'\n )\n self.test_parser.add_argument(\n '--filter', help='Filter running benchmark',\n default=None, metavar='test name'\n )\n elif self.getCmdlineValue(\"test\") == \"portChecker\":\n self.test_parser.add_argument(\n '--func_name', help='Name of open port checker function',\n default=None, metavar='func_name'\n )\n return back\n" }, { "path": "plugins/Benchmark/__init__.py", "content": "from . import BenchmarkPlugin\nfrom . import BenchmarkDb\nfrom . import BenchmarkPack\n" }, { "path": "plugins/Benchmark/media/benchmark.html", "content": "\n\n\n\n\n
Benchmark
\n\n
\n\n\n\n"
},
{
"path": "plugins/Benchmark/plugin_info.json",
"content": "{\n\t\"name\": \"Benchmark\",\n\t\"description\": \"Test and benchmark database and cryptographic functions related to ZeroNet.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/Bigfile/BigfilePiecefield.py",
"content": "import array\n\n\ndef packPiecefield(data):\n if not isinstance(data, bytes) and not isinstance(data, bytearray):\n raise Exception(\"Invalid data type: %s\" % type(data))\n\n res = []\n if not data:\n return array.array(\"H\", b\"\")\n\n if data[0] == b\"\\x00\":\n res.append(0)\n find = b\"\\x01\"\n else:\n find = b\"\\x00\"\n last_pos = 0\n pos = 0\n while 1:\n pos = data.find(find, pos)\n if find == b\"\\x00\":\n find = b\"\\x01\"\n else:\n find = b\"\\x00\"\n if pos == -1:\n res.append(len(data) - last_pos)\n break\n res.append(pos - last_pos)\n last_pos = pos\n return array.array(\"H\", res)\n\n\ndef unpackPiecefield(data):\n if not data:\n return b\"\"\n\n res = []\n char = b\"\\x01\"\n for times in data:\n if times > 10000:\n return b\"\"\n res.append(char * times)\n if char == b\"\\x01\":\n char = b\"\\x00\"\n else:\n char = b\"\\x01\"\n return b\"\".join(res)\n\n\ndef spliceBit(data, idx, bit):\n if bit != b\"\\x00\" and bit != b\"\\x01\":\n raise Exception(\"Invalid bit: %s\" % bit)\n\n if len(data) < idx:\n data = data.ljust(idx + 1, b\"\\x00\")\n return data[:idx] + bit + data[idx+ 1:]\n\nclass Piecefield(object):\n def tostring(self):\n return \"\".join([\"1\" if b else \"0\" for b in self.tobytes()])\n\n\nclass BigfilePiecefield(Piecefield):\n __slots__ = [\"data\"]\n\n def __init__(self):\n self.data = b\"\"\n\n def frombytes(self, s):\n if not isinstance(s, bytes) and not isinstance(s, bytearray):\n raise Exception(\"Invalid type: %s\" % type(s))\n self.data = s\n\n def tobytes(self):\n return self.data\n\n def pack(self):\n return packPiecefield(self.data).tobytes()\n\n def unpack(self, s):\n self.data = unpackPiecefield(array.array(\"H\", s))\n\n def __getitem__(self, key):\n try:\n return self.data[key]\n except IndexError:\n return False\n\n def __setitem__(self, key, value):\n self.data = spliceBit(self.data, key, value)\n\nclass BigfilePiecefieldPacked(Piecefield):\n __slots__ = [\"data\"]\n\n def __init__(self):\n self.data = b\"\"\n\n def frombytes(self, data):\n if not isinstance(data, bytes) and not isinstance(data, bytearray):\n raise Exception(\"Invalid type: %s\" % type(data))\n self.data = packPiecefield(data).tobytes()\n\n def tobytes(self):\n return unpackPiecefield(array.array(\"H\", self.data))\n\n def pack(self):\n return array.array(\"H\", self.data).tobytes()\n\n def unpack(self, data):\n self.data = data\n\n def __getitem__(self, key):\n try:\n return self.tobytes()[key]\n except IndexError:\n return False\n\n def __setitem__(self, key, value):\n data = spliceBit(self.tobytes(), key, value)\n self.frombytes(data)\n\n\nif __name__ == \"__main__\":\n import os\n import psutil\n import time\n testdata = b\"\\x01\" * 100 + b\"\\x00\" * 900 + b\"\\x01\" * 4000 + b\"\\x00\" * 4999 + b\"\\x01\"\n meminfo = psutil.Process(os.getpid()).memory_info\n\n for storage in [BigfilePiecefieldPacked, BigfilePiecefield]:\n print(\"-- Testing storage: %s --\" % storage)\n m = meminfo()[0]\n s = time.time()\n piecefields = {}\n for i in range(10000):\n piecefield = storage()\n piecefield.frombytes(testdata[:i] + b\"\\x00\" + testdata[i + 1:])\n piecefields[i] = piecefield\n\n print(\"Create x10000: +%sKB in %.3fs (len: %s)\" % ((meminfo()[0] - m) / 1024, time.time() - s, len(piecefields[0].data)))\n\n m = meminfo()[0]\n s = time.time()\n for piecefield in list(piecefields.values()):\n val = piecefield[1000]\n\n print(\"Query one x10000: +%sKB in %.3fs\" % ((meminfo()[0] - m) / 1024, time.time() - s))\n\n m = meminfo()[0]\n s = time.time()\n for piecefield in list(piecefields.values()):\n piecefield[1000] = b\"\\x01\"\n\n print(\"Change one x10000: +%sKB in %.3fs\" % ((meminfo()[0] - m) / 1024, time.time() - s))\n\n m = meminfo()[0]\n s = time.time()\n for piecefield in list(piecefields.values()):\n packed = piecefield.pack()\n\n print(\"Pack x10000: +%sKB in %.3fs (len: %s)\" % ((meminfo()[0] - m) / 1024, time.time() - s, len(packed)))\n\n m = meminfo()[0]\n s = time.time()\n for piecefield in list(piecefields.values()):\n piecefield.unpack(packed)\n\n print(\"Unpack x10000: +%sKB in %.3fs (len: %s)\" % ((meminfo()[0] - m) / 1024, time.time() - s, len(piecefields[0].data)))\n\n piecefields = {}\n"
},
{
"path": "plugins/Bigfile/BigfilePlugin.py",
"content": "import time\nimport os\nimport subprocess\nimport shutil\nimport collections\nimport math\nimport warnings\nimport base64\nimport binascii\nimport json\n\nimport gevent\nimport gevent.lock\n\nfrom Plugin import PluginManager\nfrom Debug import Debug\nfrom Crypt import CryptHash\nwith warnings.catch_warnings():\n warnings.filterwarnings(\"ignore\") # Ignore missing sha3 warning\n import merkletools\n\nfrom util import helper\nfrom util import Msgpack\nfrom util.Flag import flag\nimport util\nfrom .BigfilePiecefield import BigfilePiecefield, BigfilePiecefieldPacked\n\n\n# We can only import plugin host clases after the plugins are loaded\n@PluginManager.afterLoad\ndef importPluginnedClasses():\n global VerifyError, config\n from Content.ContentManager import VerifyError\n from Config import config\n\n\nif \"upload_nonces\" not in locals():\n upload_nonces = {}\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n def isCorsAllowed(self, path):\n if path == \"/ZeroNet-Internal/BigfileUpload\":\n return True\n else:\n return super(UiRequestPlugin, self).isCorsAllowed(path)\n\n @helper.encodeResponse\n def actionBigfileUpload(self):\n nonce = self.get.get(\"upload_nonce\")\n if nonce not in upload_nonces:\n return self.error403(\"Upload nonce error.\")\n\n upload_info = upload_nonces[nonce]\n del upload_nonces[nonce]\n\n self.sendHeader(200, \"text/html\", noscript=True, extra_headers={\n \"Access-Control-Allow-Origin\": \"null\",\n \"Access-Control-Allow-Credentials\": \"true\"\n })\n\n self.readMultipartHeaders(self.env['wsgi.input']) # Skip http headers\n result = self.handleBigfileUpload(upload_info, self.env['wsgi.input'].read)\n return json.dumps(result)\n\n def actionBigfileUploadWebsocket(self):\n ws = self.env.get(\"wsgi.websocket\")\n\n if not ws:\n self.start_response(\"400 Bad Request\", [])\n return [b\"Not a websocket request!\"]\n\n nonce = self.get.get(\"upload_nonce\")\n if nonce not in upload_nonces:\n return self.error403(\"Upload nonce error.\")\n\n upload_info = upload_nonces[nonce]\n del upload_nonces[nonce]\n\n ws.send(\"poll\")\n\n buffer = b\"\"\n def read(size):\n nonlocal buffer\n while len(buffer) < size:\n buffer += ws.receive()\n ws.send(\"poll\")\n part, buffer = buffer[:size], buffer[size:]\n return part\n\n result = self.handleBigfileUpload(upload_info, read)\n ws.send(json.dumps(result))\n\n def handleBigfileUpload(self, upload_info, read):\n site = upload_info[\"site\"]\n inner_path = upload_info[\"inner_path\"]\n\n with site.storage.open(inner_path, \"wb\", create_dirs=True) as out_file:\n merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile(\n read, upload_info[\"size\"], upload_info[\"piece_size\"], out_file\n )\n\n if len(piecemap_info[\"sha512_pieces\"]) == 1: # Small file, don't split\n hash = binascii.hexlify(piecemap_info[\"sha512_pieces\"][0])\n hash_id = site.content_manager.hashfield.getHashId(hash)\n site.content_manager.optionalDownloaded(inner_path, hash_id, upload_info[\"size\"], own=True)\n\n else: # Big file\n file_name = helper.getFilename(inner_path)\n site.storage.open(upload_info[\"piecemap\"], \"wb\").write(Msgpack.pack({file_name: piecemap_info}))\n\n # Find piecemap and file relative path to content.json\n file_info = site.content_manager.getFileInfo(inner_path, new_file=True)\n content_inner_path_dir = helper.getDirname(file_info[\"content_inner_path\"])\n piecemap_relative_path = upload_info[\"piecemap\"][len(content_inner_path_dir):]\n file_relative_path = inner_path[len(content_inner_path_dir):]\n\n # Add file to content.json\n if site.storage.isFile(file_info[\"content_inner_path\"]):\n content = site.storage.loadJson(file_info[\"content_inner_path\"])\n else:\n content = {}\n if \"files_optional\" not in content:\n content[\"files_optional\"] = {}\n\n content[\"files_optional\"][file_relative_path] = {\n \"sha512\": merkle_root,\n \"size\": upload_info[\"size\"],\n \"piecemap\": piecemap_relative_path,\n \"piece_size\": piece_size\n }\n\n merkle_root_hash_id = site.content_manager.hashfield.getHashId(merkle_root)\n site.content_manager.optionalDownloaded(inner_path, merkle_root_hash_id, upload_info[\"size\"], own=True)\n site.storage.writeJson(file_info[\"content_inner_path\"], content)\n\n site.content_manager.contents.loadItem(file_info[\"content_inner_path\"]) # reload cache\n\n return {\n \"merkle_root\": merkle_root,\n \"piece_num\": len(piecemap_info[\"sha512_pieces\"]),\n \"piece_size\": piece_size,\n \"inner_path\": inner_path\n }\n\n def readMultipartHeaders(self, wsgi_input):\n found = False\n for i in range(100):\n line = wsgi_input.readline()\n if line == b\"\\r\\n\":\n found = True\n break\n if not found:\n raise Exception(\"No multipart header found\")\n return i\n\n def actionFile(self, file_path, *args, **kwargs):\n if kwargs.get(\"file_size\", 0) > 1024 * 1024 and kwargs.get(\"path_parts\"): # Only check files larger than 1MB\n path_parts = kwargs[\"path_parts\"]\n site = self.server.site_manager.get(path_parts[\"address\"])\n big_file = site.storage.openBigfile(path_parts[\"inner_path\"], prebuffer=2 * 1024 * 1024)\n if big_file:\n kwargs[\"file_obj\"] = big_file\n kwargs[\"file_size\"] = big_file.size\n\n return super(UiRequestPlugin, self).actionFile(file_path, *args, **kwargs)\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def actionBigfileUploadInit(self, to, inner_path, size, protocol=\"xhr\"):\n valid_signers = self.site.content_manager.getValidSigners(inner_path)\n auth_address = self.user.getAuthAddress(self.site.address)\n if not self.site.settings[\"own\"] and auth_address not in valid_signers:\n self.log.error(\"FileWrite forbidden %s not in valid_signers %s\" % (auth_address, valid_signers))\n return self.response(to, {\"error\": \"Forbidden, you can only modify your own files\"})\n\n nonce = CryptHash.random()\n piece_size = 1024 * 1024\n inner_path = self.site.content_manager.sanitizePath(inner_path)\n file_info = self.site.content_manager.getFileInfo(inner_path, new_file=True)\n\n content_inner_path_dir = helper.getDirname(file_info[\"content_inner_path\"])\n file_relative_path = inner_path[len(content_inner_path_dir):]\n\n upload_nonces[nonce] = {\n \"added\": time.time(),\n \"site\": self.site,\n \"inner_path\": inner_path,\n \"websocket_client\": self,\n \"size\": size,\n \"piece_size\": piece_size,\n \"piecemap\": inner_path + \".piecemap.msgpack\"\n }\n\n if protocol == \"xhr\":\n return {\n \"url\": \"/ZeroNet-Internal/BigfileUpload?upload_nonce=\" + nonce,\n \"piece_size\": piece_size,\n \"inner_path\": inner_path,\n \"file_relative_path\": file_relative_path\n }\n elif protocol == \"websocket\":\n server_url = self.request.getWsServerUrl()\n if server_url:\n proto, host = server_url.split(\"://\")\n origin = proto.replace(\"http\", \"ws\") + \"://\" + host\n else:\n origin = \"{origin}\"\n return {\n \"url\": origin + \"/ZeroNet-Internal/BigfileUploadWebsocket?upload_nonce=\" + nonce,\n \"piece_size\": piece_size,\n \"inner_path\": inner_path,\n \"file_relative_path\": file_relative_path\n }\n else:\n return {\"error\": \"Unknown protocol\"}\n\n @flag.no_multiuser\n def actionSiteSetAutodownloadBigfileLimit(self, to, limit):\n permissions = self.getPermissions(to)\n if \"ADMIN\" not in permissions:\n return self.response(to, \"You don't have permission to run this command\")\n\n self.site.settings[\"autodownload_bigfile_size_limit\"] = int(limit)\n self.response(to, \"ok\")\n\n def actionFileDelete(self, to, inner_path):\n piecemap_inner_path = inner_path + \".piecemap.msgpack\"\n if self.hasFilePermission(inner_path) and self.site.storage.isFile(piecemap_inner_path):\n # Also delete .piecemap.msgpack file if exists\n self.log.debug(\"Deleting piecemap: %s\" % piecemap_inner_path)\n file_info = self.site.content_manager.getFileInfo(piecemap_inner_path)\n if file_info:\n content_json = self.site.storage.loadJson(file_info[\"content_inner_path\"])\n relative_path = file_info[\"relative_path\"]\n if relative_path in content_json.get(\"files_optional\", {}):\n del content_json[\"files_optional\"][relative_path]\n self.site.storage.writeJson(file_info[\"content_inner_path\"], content_json)\n self.site.content_manager.loadContent(file_info[\"content_inner_path\"], add_bad_files=False, force=True)\n try:\n self.site.storage.delete(piecemap_inner_path)\n except Exception as err:\n self.log.error(\"File %s delete error: %s\" % (piecemap_inner_path, err))\n\n return super(UiWebsocketPlugin, self).actionFileDelete(to, inner_path)\n\n\n@PluginManager.registerTo(\"ContentManager\")\nclass ContentManagerPlugin(object):\n def getFileInfo(self, inner_path, *args, **kwargs):\n if \"|\" not in inner_path:\n return super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs)\n\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n file_info = super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs)\n return file_info\n\n def readFile(self, read_func, size, buff_size=1024 * 64):\n part_num = 0\n recv_left = size\n\n while 1:\n part_num += 1\n read_size = min(buff_size, recv_left)\n part = read_func(read_size)\n\n if not part:\n break\n yield part\n\n if part_num % 100 == 0: # Avoid blocking ZeroNet execution during upload\n time.sleep(0.001)\n\n recv_left -= read_size\n if recv_left <= 0:\n break\n\n def hashBigfile(self, read_func, size, piece_size=1024 * 1024, file_out=None):\n self.site.settings[\"has_bigfile\"] = True\n\n recv = 0\n try:\n piece_hash = CryptHash.sha512t()\n piece_hashes = []\n piece_recv = 0\n\n mt = merkletools.MerkleTools()\n mt.hash_function = CryptHash.sha512t\n\n part = \"\"\n for part in self.readFile(read_func, size):\n if file_out:\n file_out.write(part)\n\n recv += len(part)\n piece_recv += len(part)\n piece_hash.update(part)\n if piece_recv >= piece_size:\n piece_digest = piece_hash.digest()\n piece_hashes.append(piece_digest)\n mt.leaves.append(piece_digest)\n piece_hash = CryptHash.sha512t()\n piece_recv = 0\n\n if len(piece_hashes) % 100 == 0 or recv == size:\n self.log.info(\"- [HASHING:%.0f%%] Pieces: %s, %.1fMB/%.1fMB\" % (\n float(recv) / size * 100, len(piece_hashes), recv / 1024 / 1024, size / 1024 / 1024\n ))\n part = \"\"\n if len(part) > 0:\n piece_digest = piece_hash.digest()\n piece_hashes.append(piece_digest)\n mt.leaves.append(piece_digest)\n except Exception as err:\n raise err\n finally:\n if file_out:\n file_out.close()\n\n mt.make_tree()\n merkle_root = mt.get_merkle_root()\n if type(merkle_root) is bytes: # Python <3.5\n merkle_root = merkle_root.decode()\n return merkle_root, piece_size, {\n \"sha512_pieces\": piece_hashes\n }\n\n def hashFile(self, dir_inner_path, file_relative_path, optional=False):\n inner_path = dir_inner_path + file_relative_path\n\n file_size = self.site.storage.getSize(inner_path)\n # Only care about optional files >1MB\n if not optional or file_size < 1 * 1024 * 1024:\n return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)\n\n back = {}\n content = self.contents.get(dir_inner_path + \"content.json\")\n\n hash = None\n piecemap_relative_path = None\n piece_size = None\n\n # Don't re-hash if it's already in content.json\n if content and file_relative_path in content.get(\"files_optional\", {}):\n file_node = content[\"files_optional\"][file_relative_path]\n if file_node[\"size\"] == file_size:\n self.log.info(\"- [SAME SIZE] %s\" % file_relative_path)\n hash = file_node.get(\"sha512\")\n piecemap_relative_path = file_node.get(\"piecemap\")\n piece_size = file_node.get(\"piece_size\")\n\n if not hash or not piecemap_relative_path: # Not in content.json yet\n if file_size < 5 * 1024 * 1024: # Don't create piecemap automatically for files smaller than 5MB\n return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)\n\n self.log.info(\"- [HASHING] %s\" % file_relative_path)\n merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, \"rb\").read, file_size)\n if not hash:\n hash = merkle_root\n\n if not piecemap_relative_path:\n file_name = helper.getFilename(file_relative_path)\n piecemap_relative_path = file_relative_path + \".piecemap.msgpack\"\n piecemap_inner_path = inner_path + \".piecemap.msgpack\"\n\n self.site.storage.open(piecemap_inner_path, \"wb\").write(Msgpack.pack({file_name: piecemap_info}))\n\n back.update(super(ContentManagerPlugin, self).hashFile(dir_inner_path, piecemap_relative_path, optional=True))\n\n piece_num = int(math.ceil(float(file_size) / piece_size))\n\n # Add the merkle root to hashfield\n hash_id = self.site.content_manager.hashfield.getHashId(hash)\n self.optionalDownloaded(inner_path, hash_id, file_size, own=True)\n self.site.storage.piecefields[hash].frombytes(b\"\\x01\" * piece_num)\n\n back[file_relative_path] = {\"sha512\": hash, \"size\": file_size, \"piecemap\": piecemap_relative_path, \"piece_size\": piece_size}\n return back\n\n def getPiecemap(self, inner_path):\n file_info = self.site.content_manager.getFileInfo(inner_path)\n piecemap_inner_path = helper.getDirname(file_info[\"content_inner_path\"]) + file_info[\"piecemap\"]\n self.site.needFile(piecemap_inner_path, priority=20)\n piecemap = Msgpack.unpack(self.site.storage.open(piecemap_inner_path, \"rb\").read())[helper.getFilename(inner_path)]\n piecemap[\"piece_size\"] = file_info[\"piece_size\"]\n return piecemap\n\n def verifyPiece(self, inner_path, pos, piece):\n try:\n piecemap = self.getPiecemap(inner_path)\n except Exception as err:\n raise VerifyError(\"Unable to download piecemap: %s\" % Debug.formatException(err))\n\n piece_i = int(pos / piecemap[\"piece_size\"])\n if CryptHash.sha512sum(piece, format=\"digest\") != piecemap[\"sha512_pieces\"][piece_i]:\n raise VerifyError(\"Invalid hash\")\n return True\n\n def verifyFile(self, inner_path, file, ignore_same=True):\n if \"|\" not in inner_path:\n return super(ContentManagerPlugin, self).verifyFile(inner_path, file, ignore_same)\n\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n\n return self.verifyPiece(inner_path, pos_from, file)\n\n def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):\n if \"|\" in inner_path:\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n file_info = self.getFileInfo(inner_path)\n\n # Mark piece downloaded\n piece_i = int(pos_from / file_info[\"piece_size\"])\n self.site.storage.piecefields[file_info[\"sha512\"]][piece_i] = b\"\\x01\"\n\n # Only add to site size on first request\n if hash_id in self.hashfield:\n size = 0\n elif size > 1024 * 1024:\n file_info = self.getFileInfo(inner_path)\n if file_info and \"sha512\" in file_info: # We already have the file, but not in piecefield\n sha512 = file_info[\"sha512\"]\n if sha512 not in self.site.storage.piecefields:\n self.site.storage.checkBigfile(inner_path)\n\n return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own)\n\n def optionalRemoved(self, inner_path, hash_id, size=None):\n if size and size > 1024 * 1024:\n file_info = self.getFileInfo(inner_path)\n sha512 = file_info[\"sha512\"]\n if sha512 in self.site.storage.piecefields:\n del self.site.storage.piecefields[sha512]\n\n # Also remove other pieces of the file from download queue\n for key in list(self.site.bad_files.keys()):\n if key.startswith(inner_path + \"|\"):\n del self.site.bad_files[key]\n self.site.worker_manager.removeSolvedFileTasks()\n return super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size)\n\n\n@PluginManager.registerTo(\"SiteStorage\")\nclass SiteStoragePlugin(object):\n def __init__(self, *args, **kwargs):\n super(SiteStoragePlugin, self).__init__(*args, **kwargs)\n self.piecefields = collections.defaultdict(BigfilePiecefield)\n if \"piecefields\" in self.site.settings.get(\"cache\", {}):\n for sha512, piecefield_packed in self.site.settings[\"cache\"].get(\"piecefields\").items():\n if piecefield_packed:\n self.piecefields[sha512].unpack(base64.b64decode(piecefield_packed))\n self.site.settings[\"cache\"][\"piecefields\"] = {}\n\n def createSparseFile(self, inner_path, size, sha512=None):\n file_path = self.getPath(inner_path)\n\n self.ensureDir(os.path.dirname(inner_path))\n\n f = open(file_path, 'wb')\n f.truncate(min(1024 * 1024 * 5, size)) # Only pre-allocate up to 5MB\n f.close()\n if os.name == \"nt\":\n startupinfo = subprocess.STARTUPINFO()\n startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW\n subprocess.call([\"fsutil\", \"sparse\", \"setflag\", file_path], close_fds=True, startupinfo=startupinfo)\n\n if sha512 and sha512 in self.piecefields:\n self.log.debug(\"%s: File not exists, but has piecefield. Deleting piecefield.\" % inner_path)\n del self.piecefields[sha512]\n\n def write(self, inner_path, content):\n if \"|\" not in inner_path:\n return super(SiteStoragePlugin, self).write(inner_path, content)\n\n # Write to specific position by passing |{pos} after the filename\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n file_path = self.getPath(inner_path)\n\n # Create dir if not exist\n self.ensureDir(os.path.dirname(inner_path))\n\n if not os.path.isfile(file_path):\n file_info = self.site.content_manager.getFileInfo(inner_path)\n self.createSparseFile(inner_path, file_info[\"size\"])\n\n # Write file\n with open(file_path, \"rb+\") as file:\n file.seek(pos_from)\n if hasattr(content, 'read'): # File-like object\n shutil.copyfileobj(content, file) # Write buff to disk\n else: # Simple string\n file.write(content)\n del content\n self.onUpdated(inner_path)\n\n def checkBigfile(self, inner_path):\n file_info = self.site.content_manager.getFileInfo(inner_path)\n if not file_info or (file_info and \"piecemap\" not in file_info): # It's not a big file\n return False\n\n self.site.settings[\"has_bigfile\"] = True\n file_path = self.getPath(inner_path)\n sha512 = file_info[\"sha512\"]\n piece_num = int(math.ceil(float(file_info[\"size\"]) / file_info[\"piece_size\"]))\n if os.path.isfile(file_path):\n if sha512 not in self.piecefields:\n if open(file_path, \"rb\").read(128) == b\"\\0\" * 128:\n piece_data = b\"\\x00\"\n else:\n piece_data = b\"\\x01\"\n self.log.debug(\"%s: File exists, but not in piecefield. Filling piecefiled with %s * %s.\" % (inner_path, piece_num, piece_data))\n self.piecefields[sha512].frombytes(piece_data * piece_num)\n else:\n self.log.debug(\"Creating bigfile: %s\" % inner_path)\n self.createSparseFile(inner_path, file_info[\"size\"], sha512)\n self.piecefields[sha512].frombytes(b\"\\x00\" * piece_num)\n self.log.debug(\"Created bigfile: %s\" % inner_path)\n return True\n\n def openBigfile(self, inner_path, prebuffer=0):\n if not self.checkBigfile(inner_path):\n return False\n self.site.needFile(inner_path, blocking=False) # Download piecemap\n return BigFile(self.site, inner_path, prebuffer=prebuffer)\n\n\nclass BigFile(object):\n def __init__(self, site, inner_path, prebuffer=0):\n self.site = site\n self.inner_path = inner_path\n file_path = site.storage.getPath(inner_path)\n file_info = self.site.content_manager.getFileInfo(inner_path)\n self.piece_size = file_info[\"piece_size\"]\n self.sha512 = file_info[\"sha512\"]\n self.size = file_info[\"size\"]\n self.prebuffer = prebuffer\n self.read_bytes = 0\n\n self.piecefield = self.site.storage.piecefields[self.sha512]\n self.f = open(file_path, \"rb+\")\n self.read_lock = gevent.lock.Semaphore()\n\n def read(self, buff=64 * 1024):\n with self.read_lock:\n pos = self.f.tell()\n read_until = min(self.size, pos + buff)\n requests = []\n # Request all required blocks\n while 1:\n piece_i = int(pos / self.piece_size)\n if piece_i * self.piece_size >= read_until:\n break\n pos_from = piece_i * self.piece_size\n pos_to = pos_from + self.piece_size\n if not self.piecefield[piece_i]:\n requests.append(self.site.needFile(\"%s|%s-%s\" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=10))\n pos += self.piece_size\n\n if not all(requests):\n return None\n\n # Request prebuffer\n if self.prebuffer:\n prebuffer_until = min(self.size, read_until + self.prebuffer)\n priority = 3\n while 1:\n piece_i = int(pos / self.piece_size)\n if piece_i * self.piece_size >= prebuffer_until:\n break\n pos_from = piece_i * self.piece_size\n pos_to = pos_from + self.piece_size\n if not self.piecefield[piece_i]:\n self.site.needFile(\"%s|%s-%s\" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=max(0, priority))\n priority -= 1\n pos += self.piece_size\n\n gevent.joinall(requests)\n self.read_bytes += buff\n\n # Increase buffer for long reads\n if self.read_bytes > 7 * 1024 * 1024 and self.prebuffer < 5 * 1024 * 1024:\n self.site.log.debug(\"%s: Increasing bigfile buffer size to 5MB...\" % self.inner_path)\n self.prebuffer = 5 * 1024 * 1024\n\n return self.f.read(buff)\n\n def seek(self, pos, whence=0):\n with self.read_lock:\n if whence == 2: # Relative from file end\n pos = self.size + pos # Use the real size instead of size on the disk\n whence = 0\n return self.f.seek(pos, whence)\n\n def seekable(self):\n return self.f.seekable()\n\n def tell(self):\n return self.f.tell()\n\n def close(self):\n self.f.close()\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.close()\n\n\n@PluginManager.registerTo(\"WorkerManager\")\nclass WorkerManagerPlugin(object):\n def addTask(self, inner_path, *args, **kwargs):\n file_info = kwargs.get(\"file_info\")\n if file_info and \"piecemap\" in file_info: # Bigfile\n self.site.settings[\"has_bigfile\"] = True\n\n piecemap_inner_path = helper.getDirname(file_info[\"content_inner_path\"]) + file_info[\"piecemap\"]\n piecemap_task = None\n if not self.site.storage.isFile(piecemap_inner_path):\n # Start download piecemap\n piecemap_task = super(WorkerManagerPlugin, self).addTask(piecemap_inner_path, priority=30)\n autodownload_bigfile_size_limit = self.site.settings.get(\"autodownload_bigfile_size_limit\", config.autodownload_bigfile_size_limit)\n if \"|\" not in inner_path and self.site.isDownloadable(inner_path) and file_info[\"size\"] / 1024 / 1024 <= autodownload_bigfile_size_limit:\n gevent.spawn_later(0.1, self.site.needFile, inner_path + \"|all\") # Download all pieces\n\n if \"|\" in inner_path:\n # Start download piece\n task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs)\n\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n task[\"piece_i\"] = int(pos_from / file_info[\"piece_size\"])\n task[\"sha512\"] = file_info[\"sha512\"]\n else:\n if inner_path in self.site.bad_files:\n del self.site.bad_files[inner_path]\n if piecemap_task:\n task = piecemap_task\n else:\n fake_evt = gevent.event.AsyncResult() # Don't download anything if no range specified\n fake_evt.set(True)\n task = {\"evt\": fake_evt}\n\n if not self.site.storage.isFile(inner_path):\n self.site.storage.createSparseFile(inner_path, file_info[\"size\"], file_info[\"sha512\"])\n piece_num = int(math.ceil(float(file_info[\"size\"]) / file_info[\"piece_size\"]))\n self.site.storage.piecefields[file_info[\"sha512\"]].frombytes(b\"\\x00\" * piece_num)\n else:\n task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs)\n return task\n\n def taskAddPeer(self, task, peer):\n if \"piece_i\" in task:\n if not peer.piecefields[task[\"sha512\"]][task[\"piece_i\"]]:\n if task[\"sha512\"] not in peer.piecefields:\n gevent.spawn(peer.updatePiecefields, force=True)\n elif not task[\"peers\"]:\n gevent.spawn(peer.updatePiecefields)\n\n return False # Deny to add peers to task if file not in piecefield\n return super(WorkerManagerPlugin, self).taskAddPeer(task, peer)\n\n\n@PluginManager.registerTo(\"FileRequest\")\nclass FileRequestPlugin(object):\n def isReadable(self, site, inner_path, file, pos):\n # Peek into file\n if file.read(10) == b\"\\0\" * 10:\n # Looks empty, but makes sures we don't have that piece\n file_info = site.content_manager.getFileInfo(inner_path)\n if \"piece_size\" in file_info:\n piece_i = int(pos / file_info[\"piece_size\"])\n if not site.storage.piecefields[file_info[\"sha512\"]][piece_i]:\n return False\n # Seek back to position we want to read\n file.seek(pos)\n return super(FileRequestPlugin, self).isReadable(site, inner_path, file, pos)\n\n def actionGetPiecefields(self, params):\n site = self.sites.get(params[\"site\"])\n if not site or not site.isServing(): # Site unknown or not serving\n self.response({\"error\": \"Unknown site\"})\n return False\n\n # Add peer to site if not added before\n peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True)\n if not peer.connection: # Just added\n peer.connect(self.connection) # Assign current connection to peer\n\n piecefields_packed = {sha512: piecefield.pack() for sha512, piecefield in site.storage.piecefields.items()}\n self.response({\"piecefields_packed\": piecefields_packed})\n\n def actionSetPiecefields(self, params):\n site = self.sites.get(params[\"site\"])\n if not site or not site.isServing(): # Site unknown or not serving\n self.response({\"error\": \"Unknown site\"})\n self.connection.badAction(5)\n return False\n\n # Add or get peer\n peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection)\n if not peer.connection:\n peer.connect(self.connection)\n\n peer.piecefields = collections.defaultdict(BigfilePiecefieldPacked)\n for sha512, piecefield_packed in params[\"piecefields_packed\"].items():\n peer.piecefields[sha512].unpack(piecefield_packed)\n site.settings[\"has_bigfile\"] = True\n\n self.response({\"ok\": \"Updated\"})\n\n\n@PluginManager.registerTo(\"Peer\")\nclass PeerPlugin(object):\n def __getattr__(self, key):\n if key == \"piecefields\":\n self.piecefields = collections.defaultdict(BigfilePiecefieldPacked)\n return self.piecefields\n elif key == \"time_piecefields_updated\":\n self.time_piecefields_updated = None\n return self.time_piecefields_updated\n else:\n return super(PeerPlugin, self).__getattr__(key)\n\n @util.Noparallel(ignore_args=True)\n def updatePiecefields(self, force=False):\n if self.connection and self.connection.handshake.get(\"rev\", 0) < 2190:\n return False # Not supported\n\n # Don't update piecefield again in 1 min\n if self.time_piecefields_updated and time.time() - self.time_piecefields_updated < 60 and not force:\n return False\n\n self.time_piecefields_updated = time.time()\n res = self.request(\"getPiecefields\", {\"site\": self.site.address})\n if not res or \"error\" in res:\n return False\n\n self.piecefields = collections.defaultdict(BigfilePiecefieldPacked)\n try:\n for sha512, piecefield_packed in res[\"piecefields_packed\"].items():\n self.piecefields[sha512].unpack(piecefield_packed)\n except Exception as err:\n self.log(\"Invalid updatePiecefields response: %s\" % Debug.formatException(err))\n\n return self.piecefields\n\n def sendMyHashfield(self, *args, **kwargs):\n return super(PeerPlugin, self).sendMyHashfield(*args, **kwargs)\n\n def updateHashfield(self, *args, **kwargs):\n if self.site.settings.get(\"has_bigfile\"):\n thread = gevent.spawn(self.updatePiecefields, *args, **kwargs)\n back = super(PeerPlugin, self).updateHashfield(*args, **kwargs)\n thread.join()\n return back\n else:\n return super(PeerPlugin, self).updateHashfield(*args, **kwargs)\n\n def getFile(self, site, inner_path, *args, **kwargs):\n if \"|\" in inner_path:\n inner_path, file_range = inner_path.split(\"|\")\n pos_from, pos_to = map(int, file_range.split(\"-\"))\n kwargs[\"pos_from\"] = pos_from\n kwargs[\"pos_to\"] = pos_to\n return super(PeerPlugin, self).getFile(site, inner_path, *args, **kwargs)\n\n\n@PluginManager.registerTo(\"Site\")\nclass SitePlugin(object):\n def isFileDownloadAllowed(self, inner_path, file_info):\n if \"piecemap\" in file_info:\n file_size_mb = file_info[\"size\"] / 1024 / 1024\n if config.bigfile_size_limit and file_size_mb > config.bigfile_size_limit:\n self.log.debug(\n \"Bigfile size %s too large: %sMB > %sMB, skipping...\" %\n (inner_path, file_size_mb, config.bigfile_size_limit)\n )\n return False\n\n file_info = file_info.copy()\n file_info[\"size\"] = file_info[\"piece_size\"]\n return super(SitePlugin, self).isFileDownloadAllowed(inner_path, file_info)\n\n def getSettingsCache(self):\n back = super(SitePlugin, self).getSettingsCache()\n if self.storage.piecefields:\n back[\"piecefields\"] = {sha512: base64.b64encode(piecefield.pack()).decode(\"utf8\") for sha512, piecefield in self.storage.piecefields.items()}\n return back\n\n def needFile(self, inner_path, *args, **kwargs):\n if inner_path.endswith(\"|all\"):\n @util.Pooled(20)\n def pooledNeedBigfile(inner_path, *args, **kwargs):\n if inner_path not in self.bad_files:\n self.log.debug(\"Cancelled piece, skipping %s\" % inner_path)\n return False\n return self.needFile(inner_path, *args, **kwargs)\n\n inner_path = inner_path.replace(\"|all\", \"\")\n file_info = self.needFileInfo(inner_path)\n\n # Use default function to download non-optional file\n if \"piece_size\" not in file_info:\n return super(SitePlugin, self).needFile(inner_path, *args, **kwargs)\n\n file_size = file_info[\"size\"]\n piece_size = file_info[\"piece_size\"]\n\n piece_num = int(math.ceil(float(file_size) / piece_size))\n\n file_threads = []\n\n piecefield = self.storage.piecefields.get(file_info[\"sha512\"])\n\n for piece_i in range(piece_num):\n piece_from = piece_i * piece_size\n piece_to = min(file_size, piece_from + piece_size)\n if not piecefield or not piecefield[piece_i]:\n inner_path_piece = \"%s|%s-%s\" % (inner_path, piece_from, piece_to)\n self.bad_files[inner_path_piece] = self.bad_files.get(inner_path_piece, 1)\n res = pooledNeedBigfile(inner_path_piece, blocking=False)\n if res is not True and res is not False:\n file_threads.append(res)\n gevent.joinall(file_threads)\n else:\n return super(SitePlugin, self).needFile(inner_path, *args, **kwargs)\n\n\n@PluginManager.registerTo(\"ConfigPlugin\")\nclass ConfigPlugin(object):\n def createArguments(self):\n group = self.parser.add_argument_group(\"Bigfile plugin\")\n group.add_argument('--autodownload_bigfile_size_limit', help='Also download bigfiles smaller than this limit if help distribute option is checked', default=10, metavar=\"MB\", type=int)\n group.add_argument('--bigfile_size_limit', help='Maximum size of downloaded big files', default=False, metavar=\"MB\", type=int)\n\n return super(ConfigPlugin, self).createArguments()\n"
},
{
"path": "plugins/Bigfile/Test/TestBigfile.py",
"content": "import time\nimport io\nimport binascii\n\nimport pytest\nimport mock\n\nfrom Connection import ConnectionServer\nfrom Content.ContentManager import VerifyError\nfrom File import FileServer\nfrom File import FileRequest\nfrom Worker import WorkerManager\nfrom Peer import Peer\nfrom Bigfile import BigfilePiecefield, BigfilePiecefieldPacked\nfrom Test import Spy\nfrom util import Msgpack\n\n\n@pytest.mark.usefixtures(\"resetSettings\")\n@pytest.mark.usefixtures(\"resetTempSettings\")\nclass TestBigfile:\n privatekey = \"5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv\"\n piece_size = 1024 * 1024\n\n def createBigfile(self, site, inner_path=\"data/optional.any.iso\", pieces=10):\n f = site.storage.open(inner_path, \"w\")\n for i in range(pieces * 100):\n f.write((\"Test%s\" % i).ljust(10, \"-\") * 1000)\n f.close()\n assert site.content_manager.sign(\"content.json\", self.privatekey)\n return inner_path\n\n def testPiecemapCreate(self, site):\n inner_path = self.createBigfile(site)\n content = site.storage.loadJson(\"content.json\")\n assert \"data/optional.any.iso\" in content[\"files_optional\"]\n file_node = content[\"files_optional\"][inner_path]\n assert file_node[\"size\"] == 10 * 1000 * 1000\n assert file_node[\"sha512\"] == \"47a72cde3be80b4a829e7674f72b7c6878cf6a70b0c58c6aa6c17d7e9948daf6\"\n assert file_node[\"piecemap\"] == inner_path + \".piecemap.msgpack\"\n\n piecemap = Msgpack.unpack(site.storage.open(file_node[\"piecemap\"], \"rb\").read())[\"optional.any.iso\"]\n assert len(piecemap[\"sha512_pieces\"]) == 10\n assert piecemap[\"sha512_pieces\"][0] != piecemap[\"sha512_pieces\"][1]\n assert binascii.hexlify(piecemap[\"sha512_pieces\"][0]) == b\"a73abad9992b3d0b672d0c2a292046695d31bebdcb1e150c8410bbe7c972eff3\"\n\n def testVerifyPiece(self, site):\n inner_path = self.createBigfile(site)\n\n # Verify all 10 piece\n f = site.storage.open(inner_path, \"rb\")\n for i in range(10):\n piece = io.BytesIO(f.read(1024 * 1024))\n piece.seek(0)\n site.content_manager.verifyPiece(inner_path, i * 1024 * 1024, piece)\n f.close()\n\n # Try to verify piece 0 with piece 1 hash\n with pytest.raises(VerifyError) as err:\n i = 1\n f = site.storage.open(inner_path, \"rb\")\n piece = io.BytesIO(f.read(1024 * 1024))\n f.close()\n site.content_manager.verifyPiece(inner_path, i * 1024 * 1024, piece)\n assert \"Invalid hash\" in str(err.value)\n\n def testSparseFile(self, site):\n inner_path = \"sparsefile\"\n\n # Create a 100MB sparse file\n site.storage.createSparseFile(inner_path, 100 * 1024 * 1024)\n\n # Write to file beginning\n s = time.time()\n f = site.storage.write(\"%s|%s-%s\" % (inner_path, 0, 1024 * 1024), b\"hellostart\" * 1024)\n time_write_start = time.time() - s\n\n # Write to file end\n s = time.time()\n f = site.storage.write(\"%s|%s-%s\" % (inner_path, 99 * 1024 * 1024, 99 * 1024 * 1024 + 1024 * 1024), b\"helloend\" * 1024)\n time_write_end = time.time() - s\n\n # Verify writes\n f = site.storage.open(inner_path)\n assert f.read(10) == b\"hellostart\"\n f.seek(99 * 1024 * 1024)\n assert f.read(8) == b\"helloend\"\n f.close()\n\n site.storage.delete(inner_path)\n\n # Writing to end shold not take much longer, than writing to start\n assert time_write_end <= max(0.1, time_write_start * 1.1)\n\n def testRangedFileRequest(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n file_server.sites[site.address] = site\n client = FileServer(file_server.ip, 1545)\n client.sites[site_temp.address] = site_temp\n site_temp.connection_server = client\n connection = client.getConnection(file_server.ip, 1544)\n\n # Add file_server as peer to client\n peer_file_server = site_temp.addPeer(file_server.ip, 1544)\n\n buff = peer_file_server.getFile(site_temp.address, \"%s|%s-%s\" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))\n\n assert len(buff.getvalue()) == 1 * 1024 * 1024 # Correct block size\n assert buff.getvalue().startswith(b\"Test524\") # Correct data\n buff.seek(0)\n assert site.content_manager.verifyPiece(inner_path, 5 * 1024 * 1024, buff) # Correct hash\n\n connection.close()\n client.stop()\n\n def testRangedFileDownload(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Make sure the file and the piecemap in the optional hashfield\n file_info = site.content_manager.getFileInfo(inner_path)\n assert site.content_manager.hashfield.hasHash(file_info[\"sha512\"])\n\n piecemap_hash = site.content_manager.getFileInfo(file_info[\"piecemap\"])[\"sha512\"]\n assert site.content_manager.hashfield.hasHash(piecemap_hash)\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n peer_client = site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n bad_files = site_temp.storage.verifyFiles(quick_check=True)[\"bad_files\"]\n assert not bad_files\n\n # client_piecefield = peer_client.piecefields[file_info[\"sha512\"]].tostring()\n # assert client_piecefield == \"1\" * 10\n\n # Download 5. and 10. block\n\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 9 * 1024 * 1024, 10 * 1024 * 1024))\n\n # Verify 0. block not downloaded\n f = site_temp.storage.open(inner_path)\n assert f.read(10) == b\"\\0\" * 10\n # Verify 5. and 10. block downloaded\n f.seek(5 * 1024 * 1024)\n assert f.read(7) == b\"Test524\"\n f.seek(9 * 1024 * 1024)\n assert f.read(7) == b\"943---T\"\n\n # Verify hashfield\n assert set(site_temp.content_manager.hashfield) == set([18343, 43727]) # 18343: data/optional.any.iso, 43727: data/optional.any.iso.hashmap.msgpack\n\n def testOpenBigfile(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Open virtual file\n assert not site_temp.storage.isFile(inner_path)\n\n with site_temp.storage.openBigfile(inner_path) as f:\n with Spy.Spy(FileRequest, \"route\") as requests:\n f.seek(5 * 1024 * 1024)\n assert f.read(7) == b\"Test524\"\n\n f.seek(9 * 1024 * 1024)\n assert f.read(7) == b\"943---T\"\n\n assert len(requests) == 4 # 1x peicemap + 1x getpiecefield + 2x for pieces\n\n assert set(site_temp.content_manager.hashfield) == set([18343, 43727])\n\n assert site_temp.storage.piecefields[f.sha512].tostring() == \"0000010001\"\n assert f.sha512 in site_temp.getSettingsCache()[\"piecefields\"]\n\n # Test requesting already downloaded\n with Spy.Spy(FileRequest, \"route\") as requests:\n f.seek(5 * 1024 * 1024)\n assert f.read(7) == b\"Test524\"\n\n assert len(requests) == 0\n\n # Test requesting multi-block overflow reads\n with Spy.Spy(FileRequest, \"route\") as requests:\n f.seek(5 * 1024 * 1024) # We already have this block\n data = f.read(1024 * 1024 * 3) # Our read overflow to 6. and 7. block\n assert data.startswith(b\"Test524\")\n assert data.endswith(b\"Test838-\")\n assert b\"\\0\" not in data # No null bytes allowed\n\n assert len(requests) == 2 # Two block download\n\n # Test out of range request\n f.seek(5 * 1024 * 1024)\n data = f.read(1024 * 1024 * 30)\n assert len(data) == 10 * 1000 * 1000 - (5 * 1024 * 1024)\n\n f.seek(30 * 1024 * 1024)\n data = f.read(1024 * 1024 * 30)\n assert len(data) == 0\n\n @pytest.mark.parametrize(\"piecefield_obj\", [BigfilePiecefield, BigfilePiecefieldPacked])\n def testPiecefield(self, piecefield_obj, site):\n testdatas = [\n b\"\\x01\" * 100 + b\"\\x00\" * 900 + b\"\\x01\" * 4000 + b\"\\x00\" * 4999 + b\"\\x01\",\n b\"\\x00\\x01\\x00\\x01\\x00\\x01\" * 10 + b\"\\x00\\x01\" * 90 + b\"\\x01\\x00\" * 400 + b\"\\x00\" * 4999,\n b\"\\x01\" * 10000,\n b\"\\x00\" * 10000\n ]\n for testdata in testdatas:\n piecefield = piecefield_obj()\n\n piecefield.frombytes(testdata)\n assert piecefield.tobytes() == testdata\n assert piecefield[0] == testdata[0]\n assert piecefield[100] == testdata[100]\n assert piecefield[1000] == testdata[1000]\n assert piecefield[len(testdata) - 1] == testdata[len(testdata) - 1]\n\n packed = piecefield.pack()\n piecefield_new = piecefield_obj()\n piecefield_new.unpack(packed)\n assert piecefield.tobytes() == piecefield_new.tobytes()\n assert piecefield_new.tobytes() == testdata\n\n def testFileGet(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n site_temp.connection_server = FileServer(file_server.ip, 1545)\n site_temp.connection_server.sites[site_temp.address] = site_temp\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Download second block\n with site_temp.storage.openBigfile(inner_path) as f:\n f.seek(1024 * 1024)\n assert f.read(1024)[0:1] != b\"\\0\"\n\n # Make sure first block not download\n with site_temp.storage.open(inner_path) as f:\n assert f.read(1024)[0:1] == b\"\\0\"\n\n peer2 = site.addPeer(file_server.ip, 1545, return_peer=True)\n\n # Should drop error on first block request\n assert not peer2.getFile(site.address, \"%s|0-%s\" % (inner_path, 1024 * 1024 * 1))\n\n # Should not drop error for second block request\n assert peer2.getFile(site.address, \"%s|%s-%s\" % (inner_path, 1024 * 1024 * 1, 1024 * 1024 * 2))\n\n def benchmarkPeerMemory(self, site, file_server):\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n import psutil, os\n meminfo = psutil.Process(os.getpid()).memory_info\n\n mem_s = meminfo()[0]\n s = time.time()\n for i in range(25000):\n site.addPeer(file_server.ip, i)\n print(\"%.3fs MEM: + %sKB\" % (time.time() - s, (meminfo()[0] - mem_s) / 1024)) # 0.082s MEM: + 6800KB\n print(list(site.peers.values())[0].piecefields)\n\n def testUpdatePiecefield(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n server1 = file_server\n server1.sites[site.address] = site\n server2 = FileServer(file_server.ip, 1545)\n server2.sites[site_temp.address] = site_temp\n site_temp.connection_server = server2\n\n # Add file_server as peer to client\n server2_peer1 = site_temp.addPeer(file_server.ip, 1544)\n\n # Testing piecefield sync\n assert len(server2_peer1.piecefields) == 0\n assert server2_peer1.updatePiecefields() # Query piecefields from peer\n assert len(server2_peer1.piecefields) > 0\n\n def testWorkerManagerPiecefieldDeny(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n server1 = file_server\n server1.sites[site.address] = site\n server2 = FileServer(file_server.ip, 1545)\n server2.sites[site_temp.address] = site_temp\n site_temp.connection_server = server2\n\n # Add file_server as peer to client\n server2_peer1 = site_temp.addPeer(file_server.ip, 1544) # Working\n\n site_temp.downloadContent(\"content.json\", download_files=False)\n site_temp.needFile(\"data/optional.any.iso.piecemap.msgpack\")\n\n # Add fake peers with optional files downloaded\n for i in range(5):\n fake_peer = site_temp.addPeer(\"127.0.1.%s\" % i, 1544)\n fake_peer.hashfield = site.content_manager.hashfield\n fake_peer.has_hashfield = True\n\n with Spy.Spy(WorkerManager, \"addWorker\") as requests:\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 6 * 1024 * 1024, 7 * 1024 * 1024))\n\n # It should only request parts from peer1 as the other peers does not have the requested parts in piecefields\n assert len([request[1] for request in requests if request[1] != server2_peer1]) == 0\n\n def testWorkerManagerPiecefieldDownload(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n server1 = file_server\n server1.sites[site.address] = site\n server2 = FileServer(file_server.ip, 1545)\n server2.sites[site_temp.address] = site_temp\n site_temp.connection_server = server2\n sha512 = site.content_manager.getFileInfo(inner_path)[\"sha512\"]\n\n # Create 10 fake peer for each piece\n for i in range(10):\n peer = Peer(file_server.ip, 1544, site_temp, server2)\n peer.piecefields[sha512][i] = b\"\\x01\"\n peer.updateHashfield = mock.MagicMock(return_value=False)\n peer.updatePiecefields = mock.MagicMock(return_value=False)\n peer.findHashIds = mock.MagicMock(return_value={\"nope\": []})\n peer.hashfield = site.content_manager.hashfield\n peer.has_hashfield = True\n peer.key = \"Peer:%s\" % i\n site_temp.peers[\"Peer:%s\" % i] = peer\n\n site_temp.downloadContent(\"content.json\", download_files=False)\n site_temp.needFile(\"data/optional.any.iso.piecemap.msgpack\")\n\n with Spy.Spy(Peer, \"getFile\") as requests:\n for i in range(10):\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, i * 1024 * 1024, (i + 1) * 1024 * 1024))\n\n assert len(requests) == 10\n for i in range(10):\n assert requests[i][0] == site_temp.peers[\"Peer:%s\" % i] # Every part should be requested from piece owner peer\n\n def testDownloadStats(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Open virtual file\n assert not site_temp.storage.isFile(inner_path)\n\n # Check size before downloads\n assert site_temp.settings[\"size\"] < 10 * 1024 * 1024\n assert site_temp.settings[\"optional_downloaded\"] == 0\n size_piecemap = site_temp.content_manager.getFileInfo(inner_path + \".piecemap.msgpack\")[\"size\"]\n size_bigfile = site_temp.content_manager.getFileInfo(inner_path)[\"size\"]\n\n with site_temp.storage.openBigfile(inner_path) as f:\n assert b\"\\0\" not in f.read(1024)\n assert site_temp.settings[\"optional_downloaded\"] == size_piecemap + size_bigfile\n\n with site_temp.storage.openBigfile(inner_path) as f:\n # Don't count twice\n assert b\"\\0\" not in f.read(1024)\n assert site_temp.settings[\"optional_downloaded\"] == size_piecemap + size_bigfile\n\n # Add second block\n assert b\"\\0\" not in f.read(1024 * 1024)\n assert site_temp.settings[\"optional_downloaded\"] == size_piecemap + size_bigfile\n\n def testPrebuffer(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Open virtual file\n assert not site_temp.storage.isFile(inner_path)\n\n with site_temp.storage.openBigfile(inner_path, prebuffer=1024 * 1024 * 2) as f:\n with Spy.Spy(FileRequest, \"route\") as requests:\n f.seek(5 * 1024 * 1024)\n assert f.read(7) == b\"Test524\"\n # assert len(requests) == 3 # 1x piecemap + 1x getpiecefield + 1x for pieces\n assert len([task for task in site_temp.worker_manager.tasks if task[\"inner_path\"].startswith(inner_path)]) == 2\n\n time.sleep(0.5) # Wait prebuffer download\n\n sha512 = site.content_manager.getFileInfo(inner_path)[\"sha512\"]\n assert site_temp.storage.piecefields[sha512].tostring() == \"0000011100\"\n\n # No prebuffer beyond end of the file\n f.seek(9 * 1024 * 1024)\n assert b\"\\0\" not in f.read(7)\n\n assert len([task for task in site_temp.worker_manager.tasks if task[\"inner_path\"].startswith(inner_path)]) == 0\n\n def testDownloadAllPieces(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Open virtual file\n assert not site_temp.storage.isFile(inner_path)\n\n with Spy.Spy(FileRequest, \"route\") as requests:\n site_temp.needFile(\"%s|all\" % inner_path)\n\n assert len(requests) == 12 # piecemap.msgpack, getPiecefields, 10 x piece\n\n # Don't re-download already got pieces\n with Spy.Spy(FileRequest, \"route\") as requests:\n site_temp.needFile(\"%s|all\" % inner_path)\n\n assert len(requests) == 0\n\n def testFileSize(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n client = ConnectionServer(file_server.ip, 1545)\n site_temp.connection_server = client\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n # Open virtual file\n assert not site_temp.storage.isFile(inner_path)\n\n # Download first block\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 0 * 1024 * 1024, 1 * 1024 * 1024))\n assert site_temp.storage.getSize(inner_path) < 1000 * 1000 * 10 # Size on the disk should be smaller than the real size\n\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 9 * 1024 * 1024, 10 * 1024 * 1024))\n assert site_temp.storage.getSize(inner_path) == site.storage.getSize(inner_path)\n\n def testFileRename(self, file_server, site, site_temp):\n inner_path = self.createBigfile(site)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n site_temp.connection_server = FileServer(file_server.ip, 1545)\n site_temp.connection_server.sites[site_temp.address] = site_temp\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n with Spy.Spy(FileRequest, \"route\") as requests:\n site_temp.needFile(\"%s|%s-%s\" % (inner_path, 0, 1 * self.piece_size))\n\n assert len([req for req in requests if req[1] == \"streamFile\"]) == 2 # 1 piece + piecemap\n\n # Rename the file\n inner_path_new = inner_path.replace(\".iso\", \"-new.iso\")\n site.storage.rename(inner_path, inner_path_new)\n site.storage.delete(\"data/optional.any.iso.piecemap.msgpack\")\n assert site.content_manager.sign(\"content.json\", self.privatekey, remove_missing_optional=True)\n\n files_optional = site.content_manager.contents[\"content.json\"][\"files_optional\"].keys()\n\n assert \"data/optional.any-new.iso.piecemap.msgpack\" in files_optional\n assert \"data/optional.any.iso.piecemap.msgpack\" not in files_optional\n assert \"data/optional.any.iso\" not in files_optional\n\n with Spy.Spy(FileRequest, \"route\") as requests:\n site.publish()\n time.sleep(0.1)\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10) # Wait for download\n\n assert len([req[1] for req in requests if req[1] == \"streamFile\"]) == 0\n\n with site_temp.storage.openBigfile(inner_path_new, prebuffer=0) as f:\n f.read(1024)\n\n # First piece already downloaded\n assert [req for req in requests if req[1] == \"streamFile\"] == []\n\n # Second piece needs to be downloaded + changed piecemap\n f.seek(self.piece_size)\n f.read(1024)\n assert [req[3][\"inner_path\"] for req in requests if req[1] == \"streamFile\"] == [inner_path_new + \".piecemap.msgpack\", inner_path_new]\n\n @pytest.mark.parametrize(\"size\", [1024 * 3, 1024 * 1024 * 3, 1024 * 1024 * 30])\n def testNullFileRead(self, file_server, site, site_temp, size):\n inner_path = \"data/optional.iso\"\n\n f = site.storage.open(inner_path, \"w\")\n f.write(\"\\0\" * size)\n f.close()\n assert site.content_manager.sign(\"content.json\", self.privatekey)\n\n # Init source server\n site.connection_server = file_server\n file_server.sites[site.address] = site\n\n # Init client server\n site_temp.connection_server = FileServer(file_server.ip, 1545)\n site_temp.connection_server.sites[site_temp.address] = site_temp\n site_temp.addPeer(file_server.ip, 1544)\n\n # Download site\n site_temp.download(blind_includes=True, retry_bad_files=False).join(timeout=10)\n\n if \"piecemap\" in site.content_manager.getFileInfo(inner_path): # Bigfile\n site_temp.needFile(inner_path + \"|all\")\n else:\n site_temp.needFile(inner_path)\n\n\n assert site_temp.storage.getSize(inner_path) == size\n"
},
{
"path": "plugins/Bigfile/Test/conftest.py",
"content": "from src.Test.conftest import *\n"
},
{
"path": "plugins/Bigfile/Test/pytest.ini",
"content": "[pytest]\npython_files = Test*.py\naddopts = -rsxX -v --durations=6\nmarkers =\n webtest: mark a test as a webtest."
},
{
"path": "plugins/Bigfile/__init__.py",
"content": "from . import BigfilePlugin\nfrom .BigfilePiecefield import BigfilePiecefield, BigfilePiecefieldPacked"
},
{
"path": "plugins/Chart/ChartCollector.py",
"content": "import time\nimport sys\nimport collections\nimport itertools\nimport logging\n\nimport gevent\nfrom util import helper\nfrom Config import config\n\n\nclass ChartCollector(object):\n def __init__(self, db):\n self.db = db\n if config.action == \"main\":\n gevent.spawn_later(60 * 3, self.collector)\n self.log = logging.getLogger(\"ChartCollector\")\n self.last_values = collections.defaultdict(dict)\n\n def setInitialLastValues(self, sites):\n # Recover last value of site bytes/sent\n for site in sites:\n self.last_values[\"site:\" + site.address][\"site_bytes_recv\"] = site.settings.get(\"bytes_recv\", 0)\n self.last_values[\"site:\" + site.address][\"site_bytes_sent\"] = site.settings.get(\"bytes_sent\", 0)\n\n def getCollectors(self):\n collectors = {}\n import main\n file_server = main.file_server\n sites = file_server.sites\n if not sites:\n return collectors\n content_db = list(sites.values())[0].content_manager.contents.db\n\n # Connection stats\n collectors[\"connection\"] = lambda: len(file_server.connections)\n collectors[\"connection_in\"] = (\n lambda: len([1 for connection in file_server.connections if connection.type == \"in\"])\n )\n collectors[\"connection_onion\"] = (\n lambda: len([1 for connection in file_server.connections if connection.ip.endswith(\".onion\")])\n )\n collectors[\"connection_ping_avg\"] = (\n lambda: round(1000 * helper.avg(\n [connection.last_ping_delay for connection in file_server.connections if connection.last_ping_delay]\n ))\n )\n collectors[\"connection_ping_min\"] = (\n lambda: round(1000 * min(\n [connection.last_ping_delay for connection in file_server.connections if connection.last_ping_delay]\n ))\n )\n collectors[\"connection_rev_avg\"] = (\n lambda: helper.avg(\n [connection.handshake[\"rev\"] for connection in file_server.connections if connection.handshake]\n )\n )\n\n # Request stats\n collectors[\"file_bytes_recv|change\"] = lambda: file_server.bytes_recv\n collectors[\"file_bytes_sent|change\"] = lambda: file_server.bytes_sent\n collectors[\"request_num_recv|change\"] = lambda: file_server.num_recv\n collectors[\"request_num_sent|change\"] = lambda: file_server.num_sent\n\n # Limit\n collectors[\"optional_limit\"] = lambda: content_db.getOptionalLimitBytes()\n collectors[\"optional_used\"] = lambda: content_db.getOptionalUsedBytes()\n collectors[\"optional_downloaded\"] = lambda: sum([site.settings.get(\"optional_downloaded\", 0) for site in sites.values()])\n\n # Peers\n collectors[\"peer\"] = lambda peers: len(peers)\n collectors[\"peer_onion\"] = lambda peers: len([True for peer in peers if \".onion\" in peer])\n\n # Size\n collectors[\"size\"] = lambda: sum([site.settings.get(\"size\", 0) for site in sites.values()])\n collectors[\"size_optional\"] = lambda: sum([site.settings.get(\"size_optional\", 0) for site in sites.values()])\n collectors[\"content\"] = lambda: sum([len(site.content_manager.contents) for site in sites.values()])\n\n return collectors\n\n def getSiteCollectors(self):\n site_collectors = {}\n\n # Size\n site_collectors[\"site_size\"] = lambda site: site.settings.get(\"size\", 0)\n site_collectors[\"site_size_optional\"] = lambda site: site.settings.get(\"size_optional\", 0)\n site_collectors[\"site_optional_downloaded\"] = lambda site: site.settings.get(\"optional_downloaded\", 0)\n site_collectors[\"site_content\"] = lambda site: len(site.content_manager.contents)\n\n # Data transfer\n site_collectors[\"site_bytes_recv|change\"] = lambda site: site.settings.get(\"bytes_recv\", 0)\n site_collectors[\"site_bytes_sent|change\"] = lambda site: site.settings.get(\"bytes_sent\", 0)\n\n # Peers\n site_collectors[\"site_peer\"] = lambda site: len(site.peers)\n site_collectors[\"site_peer_onion\"] = lambda site: len(\n [True for peer in site.peers.values() if peer.ip.endswith(\".onion\")]\n )\n site_collectors[\"site_peer_connected\"] = lambda site: len([True for peer in site.peers.values() if peer.connection])\n\n return site_collectors\n\n def getUniquePeers(self):\n import main\n sites = main.file_server.sites\n return set(itertools.chain.from_iterable(\n [site.peers.keys() for site in sites.values()]\n ))\n\n def collectDatas(self, collectors, last_values, site=None):\n if site is None:\n peers = self.getUniquePeers()\n datas = {}\n for key, collector in collectors.items():\n try:\n if site:\n value = collector(site)\n elif key.startswith(\"peer\"):\n value = collector(peers)\n else:\n value = collector()\n except ValueError:\n value = None\n except Exception as err:\n self.log.info(\"Collector %s error: %s\" % (key, err))\n value = None\n\n if \"|change\" in key: # Store changes relative to last value\n key = key.replace(\"|change\", \"\")\n last_value = last_values.get(key, 0)\n last_values[key] = value\n value = value - last_value\n\n if value is None:\n datas[key] = None\n else:\n datas[key] = round(value, 3)\n return datas\n\n def collectGlobal(self, collectors, last_values):\n now = int(time.time())\n s = time.time()\n datas = self.collectDatas(collectors, last_values[\"global\"])\n values = []\n for key, value in datas.items():\n values.append((self.db.getTypeId(key), value, now))\n self.log.debug(\"Global collectors done in %.3fs\" % (time.time() - s))\n\n s = time.time()\n cur = self.db.getCursor()\n cur.executemany(\"INSERT INTO data (type_id, value, date_added) VALUES (?, ?, ?)\", values)\n self.log.debug(\"Global collectors inserted in %.3fs\" % (time.time() - s))\n\n def collectSites(self, sites, collectors, last_values):\n now = int(time.time())\n s = time.time()\n values = []\n for address, site in list(sites.items()):\n site_datas = self.collectDatas(collectors, last_values[\"site:%s\" % address], site)\n for key, value in site_datas.items():\n values.append((self.db.getTypeId(key), self.db.getSiteId(address), value, now))\n time.sleep(0.001)\n self.log.debug(\"Site collections done in %.3fs\" % (time.time() - s))\n\n s = time.time()\n cur = self.db.getCursor()\n cur.executemany(\"INSERT INTO data (type_id, site_id, value, date_added) VALUES (?, ?, ?, ?)\", values)\n self.log.debug(\"Site collectors inserted in %.3fs\" % (time.time() - s))\n\n def collector(self):\n collectors = self.getCollectors()\n site_collectors = self.getSiteCollectors()\n import main\n sites = main.file_server.sites\n i = 0\n while 1:\n self.collectGlobal(collectors, self.last_values)\n if i % 12 == 0: # Only collect sites data every hour\n self.collectSites(sites, site_collectors, self.last_values)\n time.sleep(60 * 5)\n i += 1\n"
},
{
"path": "plugins/Chart/ChartDb.py",
"content": "from Config import config\nfrom Db.Db import Db\nimport time\n\n\nclass ChartDb(Db):\n def __init__(self):\n self.version = 2\n super(ChartDb, self).__init__(self.getSchema(), \"%s/chart.db\" % config.data_dir)\n self.foreign_keys = True\n self.checkTables()\n self.sites = self.loadSites()\n self.types = self.loadTypes()\n\n def getSchema(self):\n schema = {}\n schema[\"db_name\"] = \"Chart\"\n schema[\"tables\"] = {}\n schema[\"tables\"][\"data\"] = {\n \"cols\": [\n [\"data_id\", \"INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE\"],\n [\"type_id\", \"INTEGER NOT NULL\"],\n [\"site_id\", \"INTEGER\"],\n [\"value\", \"INTEGER\"],\n [\"date_added\", \"DATETIME DEFAULT (CURRENT_TIMESTAMP)\"]\n ],\n \"indexes\": [\n \"CREATE INDEX site_id ON data (site_id)\",\n \"CREATE INDEX date_added ON data (date_added)\"\n ],\n \"schema_changed\": 2\n }\n schema[\"tables\"][\"type\"] = {\n \"cols\": [\n [\"type_id\", \"INTEGER PRIMARY KEY NOT NULL UNIQUE\"],\n [\"name\", \"TEXT\"]\n ],\n \"schema_changed\": 1\n }\n schema[\"tables\"][\"site\"] = {\n \"cols\": [\n [\"site_id\", \"INTEGER PRIMARY KEY NOT NULL UNIQUE\"],\n [\"address\", \"TEXT\"]\n ],\n \"schema_changed\": 1\n }\n return schema\n\n def getTypeId(self, name):\n if name not in self.types:\n res = self.execute(\"INSERT INTO type ?\", {\"name\": name})\n self.types[name] = res.lastrowid\n\n return self.types[name]\n\n def getSiteId(self, address):\n if address not in self.sites:\n res = self.execute(\"INSERT INTO site ?\", {\"address\": address})\n self.sites[address] = res.lastrowid\n\n return self.sites[address]\n\n def loadSites(self):\n sites = {}\n for row in self.execute(\"SELECT * FROM site\"):\n sites[row[\"address\"]] = row[\"site_id\"]\n return sites\n\n def loadTypes(self):\n types = {}\n for row in self.execute(\"SELECT * FROM type\"):\n types[row[\"name\"]] = row[\"type_id\"]\n return types\n\n def deleteSite(self, address):\n if address in self.sites:\n site_id = self.sites[address]\n del self.sites[address]\n self.execute(\"DELETE FROM site WHERE ?\", {\"site_id\": site_id})\n self.execute(\"DELETE FROM data WHERE ?\", {\"site_id\": site_id})\n\n def archive(self):\n week_back = 1\n while 1:\n s = time.time()\n date_added_from = time.time() - 60 * 60 * 24 * 7 * (week_back + 1)\n date_added_to = date_added_from + 60 * 60 * 24 * 7\n res = self.execute(\"\"\"\n SELECT\n MAX(date_added) AS date_added,\n SUM(value) AS value,\n GROUP_CONCAT(data_id) AS data_ids,\n type_id,\n site_id,\n COUNT(*) AS num\n FROM data\n WHERE\n site_id IS NULL AND\n date_added > :date_added_from AND\n date_added < :date_added_to\n GROUP BY strftime('%Y-%m-%d %H', date_added, 'unixepoch', 'localtime'), type_id\n \"\"\", {\"date_added_from\": date_added_from, \"date_added_to\": date_added_to})\n\n num_archived = 0\n cur = self.getCursor()\n for row in res:\n if row[\"num\"] == 1:\n continue\n cur.execute(\"INSERT INTO data ?\", {\n \"type_id\": row[\"type_id\"],\n \"site_id\": row[\"site_id\"],\n \"value\": row[\"value\"],\n \"date_added\": row[\"date_added\"]\n })\n cur.execute(\"DELETE FROM data WHERE data_id IN (%s)\" % row[\"data_ids\"])\n num_archived += row[\"num\"]\n self.log.debug(\"Archived %s data from %s weeks ago in %.3fs\" % (num_archived, week_back, time.time() - s))\n week_back += 1\n time.sleep(0.1)\n if num_archived == 0:\n break\n # Only keep 6 month of global stats\n self.execute(\n \"DELETE FROM data WHERE site_id IS NULL AND date_added < :date_added_limit\",\n {\"date_added_limit\": time.time() - 60 * 60 * 24 * 30 * 6 }\n )\n # Only keep 1 month of site stats\n self.execute(\n \"DELETE FROM data WHERE site_id IS NOT NULL AND date_added < :date_added_limit\",\n {\"date_added_limit\": time.time() - 60 * 60 * 24 * 30 }\n )\n if week_back > 1:\n self.execute(\"VACUUM\")\n"
},
{
"path": "plugins/Chart/ChartPlugin.py",
"content": "import time\nimport itertools\n\nimport gevent\n\nfrom Config import config\nfrom util import helper\nfrom util.Flag import flag\nfrom Plugin import PluginManager\nfrom .ChartDb import ChartDb\nfrom .ChartCollector import ChartCollector\n\nif \"db\" not in locals().keys(): # Share on reloads\n db = ChartDb()\n gevent.spawn_later(10 * 60, db.archive)\n helper.timer(60 * 60 * 6, db.archive)\n collector = ChartCollector(db)\n\n@PluginManager.registerTo(\"SiteManager\")\nclass SiteManagerPlugin(object):\n def load(self, *args, **kwargs):\n back = super(SiteManagerPlugin, self).load(*args, **kwargs)\n collector.setInitialLastValues(self.sites.values())\n return back\n\n def delete(self, address, *args, **kwargs):\n db.deleteSite(address)\n return super(SiteManagerPlugin, self).delete(address, *args, **kwargs)\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n @flag.admin\n def actionChartDbQuery(self, to, query, params=None):\n if config.debug or config.verbose:\n s = time.time()\n rows = []\n try:\n if not query.strip().upper().startswith(\"SELECT\"):\n raise Exception(\"Only SELECT query supported\")\n res = db.execute(query, params)\n except Exception as err: # Response the error to client\n self.log.error(\"ChartDbQuery error: %s\" % err)\n return {\"error\": str(err)}\n # Convert result to dict\n for row in res:\n rows.append(dict(row))\n if config.verbose and time.time() - s > 0.1: # Log slow query\n self.log.debug(\"Slow query: %s (%.3fs)\" % (query, time.time() - s))\n return rows\n\n @flag.admin\n def actionChartGetPeerLocations(self, to):\n peers = {}\n for site in self.server.sites.values():\n peers.update(site.peers)\n peer_locations = self.getPeerLocations(peers)\n return peer_locations\n"
},
{
"path": "plugins/Chart/__init__.py",
"content": "from . import ChartPlugin"
},
{
"path": "plugins/Chart/plugin_info.json",
"content": "{\n\t\"name\": \"Chart\",\n\t\"description\": \"Collect and provide stats of client information.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/ContentFilter/ContentFilterPlugin.py",
"content": "import time\nimport re\nimport html\nimport os\n\nfrom Plugin import PluginManager\nfrom Translate import Translate\nfrom Config import config\nfrom util.Flag import flag\n\nfrom .ContentFilterStorage import ContentFilterStorage\n\n\nplugin_dir = os.path.dirname(__file__)\n\nif \"_\" not in locals():\n _ = Translate(plugin_dir + \"/languages/\")\n\n\n@PluginManager.registerTo(\"SiteManager\")\nclass SiteManagerPlugin(object):\n def load(self, *args, **kwargs):\n global filter_storage\n super(SiteManagerPlugin, self).load(*args, **kwargs)\n filter_storage = ContentFilterStorage(site_manager=self)\n\n def add(self, address, *args, **kwargs):\n should_ignore_block = kwargs.get(\"ignore_block\") or kwargs.get(\"settings\")\n if should_ignore_block:\n block_details = None\n elif filter_storage.isSiteblocked(address):\n block_details = filter_storage.getSiteblockDetails(address)\n else:\n address_hashed = filter_storage.getSiteAddressHashed(address)\n if filter_storage.isSiteblocked(address_hashed):\n block_details = filter_storage.getSiteblockDetails(address_hashed)\n else:\n block_details = None\n\n if block_details:\n raise Exception(\"Site blocked: %s\" % html.escape(block_details.get(\"reason\", \"unknown reason\")))\n else:\n return super(SiteManagerPlugin, self).add(address, *args, **kwargs)\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n # Mute\n def cbMuteAdd(self, to, auth_address, cert_user_id, reason):\n filter_storage.file_content[\"mutes\"][auth_address] = {\n \"cert_user_id\": cert_user_id, \"reason\": reason, \"source\": self.site.address, \"date_added\": time.time()\n }\n filter_storage.save()\n filter_storage.changeDbs(auth_address, \"remove\")\n self.response(to, \"ok\")\n\n @flag.no_multiuser\n def actionMuteAdd(self, to, auth_address, cert_user_id, reason):\n if \"ADMIN\" in self.getPermissions(to):\n self.cbMuteAdd(to, auth_address, cert_user_id, reason)\n else:\n self.cmd(\n \"confirm\",\n [_[\"Hide all content from %s?\"] % html.escape(cert_user_id), _[\"Mute\"]],\n lambda res: self.cbMuteAdd(to, auth_address, cert_user_id, reason)\n )\n\n @flag.no_multiuser\n def cbMuteRemove(self, to, auth_address):\n del filter_storage.file_content[\"mutes\"][auth_address]\n filter_storage.save()\n filter_storage.changeDbs(auth_address, \"load\")\n self.response(to, \"ok\")\n\n @flag.no_multiuser\n def actionMuteRemove(self, to, auth_address):\n if \"ADMIN\" in self.getPermissions(to):\n self.cbMuteRemove(to, auth_address)\n else:\n cert_user_id = html.escape(filter_storage.file_content[\"mutes\"][auth_address][\"cert_user_id\"])\n self.cmd(\n \"confirm\",\n [_[\"Unmute %s?\"] % cert_user_id, _[\"Unmute\"]],\n lambda res: self.cbMuteRemove(to, auth_address)\n )\n\n @flag.admin\n def actionMuteList(self, to):\n self.response(to, filter_storage.file_content[\"mutes\"])\n\n # Siteblock\n @flag.no_multiuser\n @flag.admin\n def actionSiteblockIgnoreAddSite(self, to, site_address):\n if site_address in filter_storage.site_manager.sites:\n return {\"error\": \"Site already added\"}\n else:\n if filter_storage.site_manager.need(site_address, ignore_block=True):\n return \"ok\"\n else:\n return {\"error\": \"Invalid address\"}\n\n @flag.no_multiuser\n @flag.admin\n def actionSiteblockAdd(self, to, site_address, reason=None):\n filter_storage.file_content[\"siteblocks\"][site_address] = {\"date_added\": time.time(), \"reason\": reason}\n filter_storage.save()\n self.response(to, \"ok\")\n\n @flag.no_multiuser\n @flag.admin\n def actionSiteblockRemove(self, to, site_address):\n del filter_storage.file_content[\"siteblocks\"][site_address]\n filter_storage.save()\n self.response(to, \"ok\")\n\n @flag.admin\n def actionSiteblockList(self, to):\n self.response(to, filter_storage.file_content[\"siteblocks\"])\n\n @flag.admin\n def actionSiteblockGet(self, to, site_address):\n if filter_storage.isSiteblocked(site_address):\n res = filter_storage.getSiteblockDetails(site_address)\n else:\n site_address_hashed = filter_storage.getSiteAddressHashed(site_address)\n if filter_storage.isSiteblocked(site_address_hashed):\n res = filter_storage.getSiteblockDetails(site_address_hashed)\n else:\n res = {\"error\": \"Site block not found\"}\n self.response(to, res)\n\n # Include\n @flag.no_multiuser\n def actionFilterIncludeAdd(self, to, inner_path, description=None, address=None):\n if address:\n if \"ADMIN\" not in self.getPermissions(to):\n return self.response(to, {\"error\": \"Forbidden: Only ADMIN sites can manage different site include\"})\n site = self.server.sites[address]\n else:\n address = self.site.address\n site = self.site\n\n if \"ADMIN\" in self.getPermissions(to):\n self.cbFilterIncludeAdd(to, True, address, inner_path, description)\n else:\n content = site.storage.loadJson(inner_path)\n title = _[\"New shared global content filter: %s (%s sites, %s users)\"] % (\n html.escape(inner_path), len(content.get(\"siteblocks\", {})), len(content.get(\"mutes\", {}))\n )\n\n self.cmd(\n \"confirm\",\n [title, \"Add\"],\n lambda res: self.cbFilterIncludeAdd(to, res, address, inner_path, description)\n )\n\n def cbFilterIncludeAdd(self, to, res, address, inner_path, description):\n if not res:\n self.response(to, res)\n return False\n\n filter_storage.includeAdd(address, inner_path, description)\n self.response(to, \"ok\")\n\n @flag.no_multiuser\n def actionFilterIncludeRemove(self, to, inner_path, address=None):\n if address:\n if \"ADMIN\" not in self.getPermissions(to):\n return self.response(to, {\"error\": \"Forbidden: Only ADMIN sites can manage different site include\"})\n else:\n address = self.site.address\n\n key = \"%s/%s\" % (address, inner_path)\n if key not in filter_storage.file_content[\"includes\"]:\n self.response(to, {\"error\": \"Include not found\"})\n filter_storage.includeRemove(address, inner_path)\n self.response(to, \"ok\")\n\n def actionFilterIncludeList(self, to, all_sites=False, filters=False):\n if all_sites and \"ADMIN\" not in self.getPermissions(to):\n return self.response(to, {\"error\": \"Forbidden: Only ADMIN sites can list all sites includes\"})\n\n back = []\n includes = filter_storage.file_content.get(\"includes\", {}).values()\n for include in includes:\n if not all_sites and include[\"address\"] != self.site.address:\n continue\n if filters:\n include = dict(include) # Don't modify original file_content\n include_site = filter_storage.site_manager.get(include[\"address\"])\n if not include_site:\n continue\n content = include_site.storage.loadJson(include[\"inner_path\"])\n include[\"mutes\"] = content.get(\"mutes\", {})\n include[\"siteblocks\"] = content.get(\"siteblocks\", {})\n back.append(include)\n self.response(to, back)\n\n\n@PluginManager.registerTo(\"SiteStorage\")\nclass SiteStoragePlugin(object):\n def updateDbFile(self, inner_path, file=None, cur=None):\n if file is not False: # File deletion always allowed\n # Find for bitcoin addresses in file path\n matches = re.findall(\"/(1[A-Za-z0-9]{26,35})/\", inner_path)\n # Check if any of the adresses are in the mute list\n for auth_address in matches:\n if filter_storage.isMuted(auth_address):\n self.log.debug(\"Mute match: %s, ignoring %s\" % (auth_address, inner_path))\n return False\n\n return super(SiteStoragePlugin, self).updateDbFile(inner_path, file=file, cur=cur)\n\n def onUpdated(self, inner_path, file=None):\n file_path = \"%s/%s\" % (self.site.address, inner_path)\n if file_path in filter_storage.file_content[\"includes\"]:\n self.log.debug(\"Filter file updated: %s\" % inner_path)\n filter_storage.includeUpdateAll()\n return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file)\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n def actionWrapper(self, path, extra_headers=None):\n match = re.match(r\"/(?P[A-Za-z0-9\\._-]+)(?P\n Start benchmark\n (It will take around 20 sec)\n
\n\n
\n\n\n\n\n\n\n"
},
{
"path": "plugins/ContentFilter/media/js/ZeroFrame.js",
"content": "// Version 1.0.0 - Initial release\n// Version 1.1.0 (2017-08-02) - Added cmdp function that returns promise instead of using callback\n// Version 1.2.0 (2017-08-02) - Added Ajax monkey patch to emulate XMLHttpRequest over ZeroFrame API\n\nconst CMD_INNER_READY = 'innerReady'\nconst CMD_RESPONSE = 'response'\nconst CMD_WRAPPER_READY = 'wrapperReady'\nconst CMD_PING = 'ping'\nconst CMD_PONG = 'pong'\nconst CMD_WRAPPER_OPENED_WEBSOCKET = 'wrapperOpenedWebsocket'\nconst CMD_WRAPPER_CLOSE_WEBSOCKET = 'wrapperClosedWebsocket'\n\nclass ZeroFrame {\n constructor(url) {\n this.url = url\n this.waiting_cb = {}\n this.wrapper_nonce = document.location.href.replace(/.*wrapper_nonce=([A-Za-z0-9]+).*/, \"$1\")\n this.connect()\n this.next_message_id = 1\n this.init()\n }\n\n init() {\n return this\n }\n\n connect() {\n this.target = window.parent\n window.addEventListener('message', e => this.onMessage(e), false)\n this.cmd(CMD_INNER_READY)\n }\n\n onMessage(e) {\n let message = e.data\n let cmd = message.cmd\n if (cmd === CMD_RESPONSE) {\n if (this.waiting_cb[message.to] !== undefined) {\n this.waiting_cb[message.to](message.result)\n }\n else {\n this.log(\"Websocket callback not found:\", message)\n }\n } else if (cmd === CMD_WRAPPER_READY) {\n this.cmd(CMD_INNER_READY)\n } else if (cmd === CMD_PING) {\n this.response(message.id, CMD_PONG)\n } else if (cmd === CMD_WRAPPER_OPENED_WEBSOCKET) {\n this.onOpenWebsocket()\n } else if (cmd === CMD_WRAPPER_CLOSE_WEBSOCKET) {\n this.onCloseWebsocket()\n } else {\n this.onRequest(cmd, message)\n }\n }\n\n onRequest(cmd, message) {\n this.log(\"Unknown request\", message)\n }\n\n response(to, result) {\n this.send({\n cmd: CMD_RESPONSE,\n to: to,\n result: result\n })\n }\n\n cmd(cmd, params={}, cb=null) {\n this.send({\n cmd: cmd,\n params: params\n }, cb)\n }\n\n cmdp(cmd, params={}) {\n return new Promise((resolve, reject) => {\n this.cmd(cmd, params, (res) => {\n if (res && res.error) {\n reject(res.error)\n } else {\n resolve(res)\n }\n })\n })\n }\n\n send(message, cb=null) {\n message.wrapper_nonce = this.wrapper_nonce\n message.id = this.next_message_id\n this.next_message_id++\n this.target.postMessage(message, '*')\n if (cb) {\n this.waiting_cb[message.id] = cb\n }\n }\n\n log(...args) {\n console.log.apply(console, ['[ZeroFrame]'].concat(args))\n }\n\n onOpenWebsocket() {\n this.log('Websocket open')\n }\n\n onCloseWebsocket() {\n this.log('Websocket close')\n }\n\n monkeyPatchAjax() {\n var page = this\n XMLHttpRequest.prototype.realOpen = XMLHttpRequest.prototype.open\n this.cmd(\"wrapperGetAjaxKey\", [], (res) => { this.ajax_key = res })\n var newOpen = function (method, url, async) {\n url += \"?ajax_key=\" + page.ajax_key\n return this.realOpen(method, url, async)\n }\n XMLHttpRequest.prototype.open = newOpen\n }\n}\n"
},
{
"path": "plugins/ContentFilter/plugin_info.json",
"content": "{\n\t\"name\": \"ContentFilter\",\n\t\"description\": \"Manage site and user block list.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/Cors/CorsPlugin.py",
"content": "import re\nimport html\nimport copy\nimport os\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Translate import Translate\n\n\nplugin_dir = os.path.dirname(__file__)\n\nif \"_\" not in locals():\n _ = Translate(plugin_dir + \"/languages/\")\n\n\ndef getCorsPath(site, inner_path):\n match = re.match(\"^cors-([A-Za-z0-9]{26,35})/(.*)\", inner_path)\n if not match:\n raise Exception(\"Invalid cors path: %s\" % inner_path)\n cors_address = match.group(1)\n cors_inner_path = match.group(2)\n\n if not \"Cors:%s\" % cors_address in site.settings[\"permissions\"]:\n raise Exception(\"This site has no permission to access site %s\" % cors_address)\n\n return cors_address, cors_inner_path\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def hasSitePermission(self, address, cmd=None):\n if super(UiWebsocketPlugin, self).hasSitePermission(address, cmd=cmd):\n return True\n\n allowed_commands = [\n \"fileGet\", \"fileList\", \"dirList\", \"fileRules\", \"optionalFileInfo\",\n \"fileQuery\", \"dbQuery\", \"userGetSettings\", \"siteInfo\"\n ]\n if not \"Cors:%s\" % address in self.site.settings[\"permissions\"] or cmd not in allowed_commands:\n return False\n else:\n return True\n\n # Add cors support for file commands\n def corsFuncWrapper(self, func_name, to, inner_path, *args, **kwargs):\n if inner_path.startswith(\"cors-\"):\n cors_address, cors_inner_path = getCorsPath(self.site, inner_path)\n\n req_self = copy.copy(self)\n req_self.site = self.server.sites.get(cors_address) # Change the site to the merged one\n if not req_self.site:\n return {\"error\": \"No site found\"}\n\n func = getattr(super(UiWebsocketPlugin, req_self), func_name)\n back = func(to, cors_inner_path, *args, **kwargs)\n return back\n else:\n func = getattr(super(UiWebsocketPlugin, self), func_name)\n return func(to, inner_path, *args, **kwargs)\n\n def actionFileGet(self, to, inner_path, *args, **kwargs):\n return self.corsFuncWrapper(\"actionFileGet\", to, inner_path, *args, **kwargs)\n\n def actionFileList(self, to, inner_path, *args, **kwargs):\n return self.corsFuncWrapper(\"actionFileList\", to, inner_path, *args, **kwargs)\n\n def actionDirList(self, to, inner_path, *args, **kwargs):\n return self.corsFuncWrapper(\"actionDirList\", to, inner_path, *args, **kwargs)\n\n def actionFileRules(self, to, inner_path, *args, **kwargs):\n return self.corsFuncWrapper(\"actionFileRules\", to, inner_path, *args, **kwargs)\n\n def actionOptionalFileInfo(self, to, inner_path, *args, **kwargs):\n return self.corsFuncWrapper(\"actionOptionalFileInfo\", to, inner_path, *args, **kwargs)\n\n def actionCorsPermission(self, to, address):\n if isinstance(address, list):\n addresses = address\n else:\n addresses = [address]\n\n button_title = _[\"Grant\"]\n site_names = []\n site_addresses = []\n for address in addresses:\n site = self.server.sites.get(address)\n if site:\n site_name = site.content_manager.contents.get(\"content.json\", {}).get(\"title\", address)\n else:\n site_name = address\n # If at least one site is not downloaded yet, show \"Grant & Add\" instead\n button_title = _[\"Grant & Add\"]\n\n if not (site and \"Cors:\" + address in self.permissions):\n # No site or no permission\n site_names.append(site_name)\n site_addresses.append(address)\n\n if len(site_names) == 0:\n return \"ignored\"\n\n self.cmd(\n \"confirm\",\n [_[\"This site requests read permission to: %s\"] % \", \".join(map(html.escape, site_names)), button_title],\n lambda res: self.cbCorsPermission(to, site_addresses)\n )\n\n def cbCorsPermission(self, to, addresses):\n # Add permissions\n for address in addresses:\n permission = \"Cors:\" + address\n if permission not in self.site.settings[\"permissions\"]:\n self.site.settings[\"permissions\"].append(permission)\n\n self.site.saveSettings()\n self.site.updateWebsocket(permission_added=permission)\n\n self.response(to, \"ok\")\n\n for address in addresses:\n site = self.server.sites.get(address)\n if not site:\n gevent.spawn(self.server.site_manager.need, address)\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n # Allow to load cross origin files using /cors-address/file.jpg\n def parsePath(self, path):\n path_parts = super(UiRequestPlugin, self).parsePath(path)\n if \"cors-\" not in path: # Optimization\n return path_parts\n site = self.server.sites[path_parts[\"address\"]]\n try:\n path_parts[\"address\"], path_parts[\"inner_path\"] = getCorsPath(site, path_parts[\"inner_path\"])\n except Exception:\n return None\n return path_parts\n"
},
{
"path": "plugins/Cors/__init__.py",
"content": "from . import CorsPlugin"
},
{
"path": "plugins/Cors/plugin_info.json",
"content": "{\n\t\"name\": \"Cors\",\n\t\"description\": \"Cross site resource read.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/CryptMessage/CryptMessage.py",
"content": "import hashlib\nimport base64\nimport struct\nfrom lib import sslcrypto\nfrom Crypt import Crypt\n\n\ncurve = sslcrypto.ecc.get_curve(\"secp256k1\")\n\n\ndef eciesEncrypt(data, pubkey, ciphername=\"aes-256-cbc\"):\n ciphertext, key_e = curve.encrypt(\n data,\n base64.b64decode(pubkey),\n algo=ciphername,\n derivation=\"sha512\",\n return_aes_key=True\n )\n return key_e, ciphertext\n\n\n@Crypt.thread_pool_crypt.wrap\ndef eciesDecryptMulti(encrypted_datas, privatekey):\n texts = [] # Decoded texts\n for encrypted_data in encrypted_datas:\n try:\n text = eciesDecrypt(encrypted_data, privatekey).decode(\"utf8\")\n texts.append(text)\n except Exception:\n texts.append(None)\n return texts\n\n\ndef eciesDecrypt(ciphertext, privatekey):\n return curve.decrypt(base64.b64decode(ciphertext), curve.wif_to_private(privatekey.encode()), derivation=\"sha512\")\n\n\ndef decodePubkey(pubkey):\n i = 0\n curve = struct.unpack('!H', pubkey[i:i + 2])[0]\n i += 2\n tmplen = struct.unpack('!H', pubkey[i:i + 2])[0]\n i += 2\n pubkey_x = pubkey[i:i + tmplen]\n i += tmplen\n tmplen = struct.unpack('!H', pubkey[i:i + 2])[0]\n i += 2\n pubkey_y = pubkey[i:i + tmplen]\n i += tmplen\n return curve, pubkey_x, pubkey_y, i\n\n\ndef split(encrypted):\n iv = encrypted[0:16]\n curve, pubkey_x, pubkey_y, i = decodePubkey(encrypted[16:])\n ciphertext = encrypted[16 + i:-32]\n\n return iv, ciphertext\n"
},
{
"path": "plugins/CryptMessage/CryptMessagePlugin.py",
"content": "import base64\nimport os\n\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Crypt import CryptBitcoin, CryptHash\nfrom Config import config\nimport sslcrypto\n\nfrom . import CryptMessage\n\ncurve = sslcrypto.ecc.get_curve(\"secp256k1\")\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n # - Actions -\n\n # Returns user's public key unique to site\n # Return: Public key\n def actionUserPublickey(self, to, index=0):\n self.response(to, self.user.getEncryptPublickey(self.site.address, index))\n\n # Encrypt a text using the publickey or user's sites unique publickey\n # Return: Encrypted text using base64 encoding\n def actionEciesEncrypt(self, to, text, publickey=0, return_aes_key=False):\n if type(publickey) is int: # Encrypt using user's publickey\n publickey = self.user.getEncryptPublickey(self.site.address, publickey)\n aes_key, encrypted = CryptMessage.eciesEncrypt(text.encode(\"utf8\"), publickey)\n if return_aes_key:\n self.response(to, [base64.b64encode(encrypted).decode(\"utf8\"), base64.b64encode(aes_key).decode(\"utf8\")])\n else:\n self.response(to, base64.b64encode(encrypted).decode(\"utf8\"))\n\n # Decrypt a text using privatekey or the user's site unique private key\n # Return: Decrypted text or list of decrypted texts\n def actionEciesDecrypt(self, to, param, privatekey=0):\n if type(privatekey) is int: # Decrypt using user's privatekey\n privatekey = self.user.getEncryptPrivatekey(self.site.address, privatekey)\n\n if type(param) == list:\n encrypted_texts = param\n else:\n encrypted_texts = [param]\n\n texts = CryptMessage.eciesDecryptMulti(encrypted_texts, privatekey)\n\n if type(param) == list:\n self.response(to, texts)\n else:\n self.response(to, texts[0])\n\n # Encrypt a text using AES\n # Return: Iv, AES key, Encrypted text\n def actionAesEncrypt(self, to, text, key=None):\n if key:\n key = base64.b64decode(key)\n else:\n key = sslcrypto.aes.new_key()\n\n if text:\n encrypted, iv = sslcrypto.aes.encrypt(text.encode(\"utf8\"), key)\n else:\n encrypted, iv = b\"\", b\"\"\n\n res = [base64.b64encode(item).decode(\"utf8\") for item in [key, iv, encrypted]]\n self.response(to, res)\n\n # Decrypt a text using AES\n # Return: Decrypted text\n def actionAesDecrypt(self, to, *args):\n if len(args) == 3: # Single decrypt\n encrypted_texts = [(args[0], args[1])]\n keys = [args[2]]\n else: # Batch decrypt\n encrypted_texts, keys = args\n\n texts = [] # Decoded texts\n for iv, encrypted_text in encrypted_texts:\n encrypted_text = base64.b64decode(encrypted_text)\n iv = base64.b64decode(iv)\n text = None\n for key in keys:\n try:\n decrypted = sslcrypto.aes.decrypt(encrypted_text, iv, base64.b64decode(key))\n if decrypted and decrypted.decode(\"utf8\"): # Valid text decoded\n text = decrypted.decode(\"utf8\")\n except Exception as err:\n pass\n texts.append(text)\n\n if len(args) == 3:\n self.response(to, texts[0])\n else:\n self.response(to, texts)\n\n # Sign data using ECDSA\n # Return: Signature\n def actionEcdsaSign(self, to, data, privatekey=None):\n if privatekey is None: # Sign using user's privatekey\n privatekey = self.user.getAuthPrivatekey(self.site.address)\n\n self.response(to, CryptBitcoin.sign(data, privatekey))\n\n # Verify data using ECDSA (address is either a address or array of addresses)\n # Return: bool\n def actionEcdsaVerify(self, to, data, address, signature):\n self.response(to, CryptBitcoin.verify(data, address, signature))\n\n # Gets the publickey of a given privatekey\n def actionEccPrivToPub(self, to, privatekey):\n self.response(to, curve.private_to_public(curve.wif_to_private(privatekey.encode())))\n\n # Gets the address of a given publickey\n def actionEccPubToAddr(self, to, publickey):\n self.response(to, curve.public_to_address(bytes.fromhex(publickey)))\n\n\n@PluginManager.registerTo(\"User\")\nclass UserPlugin(object):\n def getEncryptPrivatekey(self, address, param_index=0):\n if param_index < 0 or param_index > 1000:\n raise Exception(\"Param_index out of range\")\n\n site_data = self.getSiteData(address)\n\n if site_data.get(\"cert\"): # Different privatekey for different cert provider\n index = param_index + self.getAddressAuthIndex(site_data[\"cert\"])\n else:\n index = param_index\n\n if \"encrypt_privatekey_%s\" % index not in site_data:\n address_index = self.getAddressAuthIndex(address)\n crypt_index = address_index + 1000 + index\n site_data[\"encrypt_privatekey_%s\" % index] = CryptBitcoin.hdPrivatekey(self.master_seed, crypt_index)\n self.log.debug(\"New encrypt privatekey generated for %s:%s\" % (address, index))\n return site_data[\"encrypt_privatekey_%s\" % index]\n\n def getEncryptPublickey(self, address, param_index=0):\n if param_index < 0 or param_index > 1000:\n raise Exception(\"Param_index out of range\")\n\n site_data = self.getSiteData(address)\n\n if site_data.get(\"cert\"): # Different privatekey for different cert provider\n index = param_index + self.getAddressAuthIndex(site_data[\"cert\"])\n else:\n index = param_index\n\n if \"encrypt_publickey_%s\" % index not in site_data:\n privatekey = self.getEncryptPrivatekey(address, param_index).encode()\n publickey = curve.private_to_public(curve.wif_to_private(privatekey) + b\"\\x01\")\n site_data[\"encrypt_publickey_%s\" % index] = base64.b64encode(publickey).decode(\"utf8\")\n return site_data[\"encrypt_publickey_%s\" % index]\n\n\n@PluginManager.registerTo(\"Actions\")\nclass ActionsPlugin:\n publickey = \"A3HatibU4S6eZfIQhVs2u7GLN5G9wXa9WwlkyYIfwYaj\"\n privatekey = \"5JBiKFYBm94EUdbxtnuLi6cvNcPzcKymCUHBDf2B6aq19vvG3rL\"\n utf8_text = '\\xc1rv\\xedzt\\xfbr\\xf5t\\xfck\\xf6rf\\xfar\\xf3g\\xe9p'\n\n def getBenchmarkTests(self, online=False):\n if hasattr(super(), \"getBenchmarkTests\"):\n tests = super().getBenchmarkTests(online)\n else:\n tests = []\n\n aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode(\"utf8\"), self.publickey) # Warm-up\n tests.extend([\n {\"func\": self.testCryptEciesEncrypt, \"kwargs\": {}, \"num\": 100, \"time_standard\": 1.2},\n {\"func\": self.testCryptEciesDecrypt, \"kwargs\": {}, \"num\": 500, \"time_standard\": 1.3},\n {\"func\": self.testCryptEciesDecryptMulti, \"kwargs\": {}, \"num\": 5, \"time_standard\": 0.68},\n {\"func\": self.testCryptAesEncrypt, \"kwargs\": {}, \"num\": 10000, \"time_standard\": 0.27},\n {\"func\": self.testCryptAesDecrypt, \"kwargs\": {}, \"num\": 10000, \"time_standard\": 0.25}\n ])\n return tests\n\n def testCryptEciesEncrypt(self, num_run=1):\n for i in range(num_run):\n aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode(\"utf8\"), self.publickey)\n assert len(aes_key) == 32\n yield \".\"\n\n def testCryptEciesDecrypt(self, num_run=1):\n aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode(\"utf8\"), self.publickey)\n for i in range(num_run):\n assert len(aes_key) == 32\n decrypted = CryptMessage.eciesDecrypt(base64.b64encode(encrypted), self.privatekey)\n assert decrypted == self.utf8_text.encode(\"utf8\"), \"%s != %s\" % (decrypted, self.utf8_text.encode(\"utf8\"))\n yield \".\"\n\n def testCryptEciesDecryptMulti(self, num_run=1):\n yield \"x 100 (%s threads) \" % config.threads_crypt\n aes_key, encrypted = CryptMessage.eciesEncrypt(self.utf8_text.encode(\"utf8\"), self.publickey)\n\n threads = []\n for i in range(num_run):\n assert len(aes_key) == 32\n threads.append(gevent.spawn(\n CryptMessage.eciesDecryptMulti, [base64.b64encode(encrypted)] * 100, self.privatekey\n ))\n\n for thread in threads:\n res = thread.get()\n assert res[0] == self.utf8_text, \"%s != %s\" % (res[0], self.utf8_text)\n assert res[0] == res[-1], \"%s != %s\" % (res[0], res[-1])\n yield \".\"\n gevent.joinall(threads)\n\n def testCryptAesEncrypt(self, num_run=1):\n for i in range(num_run):\n key = os.urandom(32)\n encrypted = sslcrypto.aes.encrypt(self.utf8_text.encode(\"utf8\"), key)\n yield \".\"\n\n def testCryptAesDecrypt(self, num_run=1):\n key = os.urandom(32)\n encrypted_text, iv = sslcrypto.aes.encrypt(self.utf8_text.encode(\"utf8\"), key)\n\n for i in range(num_run):\n decrypted = sslcrypto.aes.decrypt(encrypted_text, iv, key).decode(\"utf8\")\n assert decrypted == self.utf8_text\n yield \".\"\n"
},
{
"path": "plugins/CryptMessage/Test/TestCrypt.py",
"content": "import pytest\nimport base64\nfrom CryptMessage import CryptMessage\n\n\n@pytest.mark.usefixtures(\"resetSettings\")\nclass TestCrypt:\n publickey = \"A3HatibU4S6eZfIQhVs2u7GLN5G9wXa9WwlkyYIfwYaj\"\n privatekey = \"5JBiKFYBm94EUdbxtnuLi6cvNcPzcKymCUHBDf2B6aq19vvG3rL\"\n utf8_text = '\\xc1rv\\xedzt\\xfbr\\xf5t\\xfck\\xf6rf\\xfar\\xf3g\\xe9'\n ecies_encrypted_text = \"R5J1RFIDOzE5bnWopvccmALKACCk/CRcd/KSE9OgExJKASyMbZ57JVSUenL2TpABMmcT+wAgr2UrOqClxpOWvIUwvwwupXnMbRTzthhIJJrTRW3sCJVaYlGEMn9DAcvbflgEkQX/MVVdLV3tWKySs1Vk8sJC/y+4pGYCrZz7vwDNEEERaqU=\"\n\n @pytest.mark.parametrize(\"text\", [b\"hello\", '\\xc1rv\\xedzt\\xfbr\\xf5t\\xfck\\xf6rf\\xfar\\xf3g\\xe9'.encode(\"utf8\")])\n @pytest.mark.parametrize(\"text_repeat\", [1, 10, 128, 1024])\n def testEncryptEcies(self, text, text_repeat):\n text_repeated = text * text_repeat\n aes_key, encrypted = CryptMessage.eciesEncrypt(text_repeated, self.publickey)\n assert len(aes_key) == 32\n # assert len(encrypted) == 134 + int(len(text) / 16) * 16 # Not always true\n\n assert CryptMessage.eciesDecrypt(base64.b64encode(encrypted), self.privatekey) == text_repeated\n\n def testDecryptEcies(self, user):\n assert CryptMessage.eciesDecrypt(self.ecies_encrypted_text, self.privatekey) == b\"hello\"\n\n def testPublickey(self, ui_websocket):\n pub = ui_websocket.testAction(\"UserPublickey\", 0)\n assert len(pub) == 44 # Compressed, b64 encoded publickey\n\n # Different pubkey for specificed index\n assert ui_websocket.testAction(\"UserPublickey\", 1) != ui_websocket.testAction(\"UserPublickey\", 0)\n\n # Same publickey for same index\n assert ui_websocket.testAction(\"UserPublickey\", 2) == ui_websocket.testAction(\"UserPublickey\", 2)\n\n # Different publickey for different cert\n site_data = ui_websocket.user.getSiteData(ui_websocket.site.address)\n site_data[\"cert\"] = None\n pub1 = ui_websocket.testAction(\"UserPublickey\", 0)\n\n site_data = ui_websocket.user.getSiteData(ui_websocket.site.address)\n site_data[\"cert\"] = \"zeroid.bit\"\n pub2 = ui_websocket.testAction(\"UserPublickey\", 0)\n assert pub1 != pub2\n\n def testEcies(self, ui_websocket):\n pub = ui_websocket.testAction(\"UserPublickey\")\n\n encrypted = ui_websocket.testAction(\"EciesEncrypt\", \"hello\", pub)\n assert len(encrypted) == 180\n\n # Don't allow decrypt using other privatekey index\n decrypted = ui_websocket.testAction(\"EciesDecrypt\", encrypted, 123)\n assert decrypted != \"hello\"\n\n # Decrypt using correct privatekey\n decrypted = ui_websocket.testAction(\"EciesDecrypt\", encrypted)\n assert decrypted == \"hello\"\n\n # Decrypt incorrect text\n decrypted = ui_websocket.testAction(\"EciesDecrypt\", \"baad\")\n assert decrypted is None\n\n # Decrypt batch\n decrypted = ui_websocket.testAction(\"EciesDecrypt\", [encrypted, \"baad\", encrypted])\n assert decrypted == [\"hello\", None, \"hello\"]\n\n def testEciesUtf8(self, ui_websocket):\n # Utf8 test\n ui_websocket.actionEciesEncrypt(0, self.utf8_text)\n encrypted = ui_websocket.ws.getResult()\n\n ui_websocket.actionEciesDecrypt(0, encrypted)\n assert ui_websocket.ws.getResult() == self.utf8_text\n\n def testEciesAes(self, ui_websocket):\n ui_websocket.actionEciesEncrypt(0, \"hello\", return_aes_key=True)\n ecies_encrypted, aes_key = ui_websocket.ws.getResult()\n\n # Decrypt using Ecies\n ui_websocket.actionEciesDecrypt(0, ecies_encrypted)\n assert ui_websocket.ws.getResult() == \"hello\"\n\n # Decrypt using AES\n aes_iv, aes_encrypted = CryptMessage.split(base64.b64decode(ecies_encrypted))\n\n ui_websocket.actionAesDecrypt(0, base64.b64encode(aes_iv), base64.b64encode(aes_encrypted), aes_key)\n assert ui_websocket.ws.getResult() == \"hello\"\n\n def testEciesAesLongpubkey(self, ui_websocket):\n privatekey = \"5HwVS1bTFnveNk9EeGaRenWS1QFzLFb5kuncNbiY3RiHZrVR6ok\"\n\n ecies_encrypted, aes_key = [\"lWiXfEikIjw1ac3J/RaY/gLKACALRUfksc9rXYRFyKDSaxhwcSFBYCgAdIyYlY294g/6VgAf/68PYBVMD3xKH1n7Zbo+ge8b4i/XTKmCZRJvy0eutMKWckYCMVcxgIYNa/ZL1BY1kvvH7omgzg1wBraoLfdbNmVtQgdAZ9XS8PwRy6OB2Q==\", \"Rvlf7zsMuBFHZIGHcbT1rb4If+YTmsWDv6kGwcvSeMM=\"]\n\n # Decrypt using Ecies\n ui_websocket.actionEciesDecrypt(0, ecies_encrypted, privatekey)\n assert ui_websocket.ws.getResult() == \"hello\"\n\n # Decrypt using AES\n aes_iv, aes_encrypted = CryptMessage.split(base64.b64decode(ecies_encrypted))\n\n ui_websocket.actionAesDecrypt(0, base64.b64encode(aes_iv), base64.b64encode(aes_encrypted), aes_key)\n assert ui_websocket.ws.getResult() == \"hello\"\n\n def testAes(self, ui_websocket):\n ui_websocket.actionAesEncrypt(0, \"hello\")\n key, iv, encrypted = ui_websocket.ws.getResult()\n\n assert len(key) == 44\n assert len(iv) == 24\n assert len(encrypted) == 24\n\n # Single decrypt\n ui_websocket.actionAesDecrypt(0, iv, encrypted, key)\n assert ui_websocket.ws.getResult() == \"hello\"\n\n # Batch decrypt\n ui_websocket.actionAesEncrypt(0, \"hello\")\n key2, iv2, encrypted2 = ui_websocket.ws.getResult()\n\n assert [key, iv, encrypted] != [key2, iv2, encrypted2]\n\n # 2 correct key\n ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, \"baad\"], [iv2, encrypted2]], [key])\n assert ui_websocket.ws.getResult() == [\"hello\", \"hello\", None, None]\n\n # 3 key\n ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, \"baad\"], [iv2, encrypted2]], [key, key2])\n assert ui_websocket.ws.getResult() == [\"hello\", \"hello\", None, \"hello\"]\n\n def testAesUtf8(self, ui_websocket):\n ui_websocket.actionAesEncrypt(0, self.utf8_text)\n key, iv, encrypted = ui_websocket.ws.getResult()\n\n ui_websocket.actionAesDecrypt(0, iv, encrypted, key)\n assert ui_websocket.ws.getResult() == self.utf8_text\n"
},
{
"path": "plugins/CryptMessage/Test/conftest.py",
"content": "from src.Test.conftest import *"
},
{
"path": "plugins/CryptMessage/Test/pytest.ini",
"content": "[pytest]\npython_files = Test*.py\naddopts = -rsxX -v --durations=6\nmarkers =\n webtest: mark a test as a webtest."
},
{
"path": "plugins/CryptMessage/__init__.py",
"content": "from . import CryptMessagePlugin"
},
{
"path": "plugins/CryptMessage/plugin_info.json",
"content": "{\n\t\"name\": \"CryptMessage\",\n\t\"description\": \"Cryptographic functions of ECIES and AES data encryption/decryption.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/FilePack/FilePackPlugin.py",
"content": "import os\nimport re\n\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom Debug import Debug\n\n# Keep archive open for faster reponse times for large sites\narchive_cache = {}\n\n\ndef closeArchive(archive_path):\n if archive_path in archive_cache:\n del archive_cache[archive_path]\n\n\ndef openArchive(archive_path, file_obj=None):\n if archive_path not in archive_cache:\n if archive_path.endswith(\"tar.gz\"):\n import tarfile\n archive_cache[archive_path] = tarfile.open(archive_path, fileobj=file_obj, mode=\"r:gz\")\n else:\n import zipfile\n archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path)\n gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec\n\n archive = archive_cache[archive_path]\n return archive\n\n\ndef openArchiveFile(archive_path, path_within, file_obj=None):\n archive = openArchive(archive_path, file_obj=file_obj)\n if archive_path.endswith(\".zip\"):\n return archive.open(path_within)\n else:\n return archive.extractfile(path_within)\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n def actionSiteMedia(self, path, **kwargs):\n if \".zip/\" in path or \".tar.gz/\" in path:\n file_obj = None\n path_parts = self.parsePath(path)\n file_path = \"%s/%s/%s\" % (config.data_dir, path_parts[\"address\"], path_parts[\"inner_path\"])\n match = re.match(\"^(.*\\.(?:tar.gz|zip))/(.*)\", file_path)\n archive_path, path_within = match.groups()\n if archive_path not in archive_cache:\n site = self.server.site_manager.get(path_parts[\"address\"])\n if not site:\n return self.actionSiteAddPrompt(path)\n archive_inner_path = site.storage.getInnerPath(archive_path)\n if not os.path.isfile(archive_path):\n # Wait until file downloads\n result = site.needFile(archive_inner_path, priority=10)\n # Send virutal file path download finished event to remove loading screen\n site.updateWebsocket(file_done=archive_inner_path)\n if not result:\n return self.error404(archive_inner_path)\n file_obj = site.storage.openBigfile(archive_inner_path)\n if file_obj == False:\n file_obj = None\n\n header_allow_ajax = False\n if self.get.get(\"ajax_key\"):\n requester_site = self.server.site_manager.get(path_parts[\"request_address\"])\n if self.get[\"ajax_key\"] == requester_site.settings[\"ajax_key\"]:\n header_allow_ajax = True\n else:\n return self.error403(\"Invalid ajax_key\")\n\n try:\n file = openArchiveFile(archive_path, path_within, file_obj=file_obj)\n content_type = self.getContentType(file_path)\n self.sendHeader(200, content_type=content_type, noscript=kwargs.get(\"header_noscript\", False), allow_ajax=header_allow_ajax)\n return self.streamFile(file)\n except Exception as err:\n self.log.debug(\"Error opening archive file: %s\" % Debug.formatException(err))\n return self.error404(path)\n\n return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)\n\n def streamFile(self, file):\n for i in range(100): # Read max 6MB\n try:\n block = file.read(60 * 1024)\n if block:\n yield block\n else:\n raise StopIteration\n except StopIteration:\n file.close()\n break\n\n\n@PluginManager.registerTo(\"SiteStorage\")\nclass SiteStoragePlugin(object):\n def isFile(self, inner_path):\n if \".zip/\" in inner_path or \".tar.gz/\" in inner_path:\n match = re.match(\"^(.*\\.(?:tar.gz|zip))/(.*)\", inner_path)\n archive_inner_path, path_within = match.groups()\n return super(SiteStoragePlugin, self).isFile(archive_inner_path)\n else:\n return super(SiteStoragePlugin, self).isFile(inner_path)\n\n def openArchive(self, inner_path):\n archive_path = self.getPath(inner_path)\n file_obj = None\n if archive_path not in archive_cache:\n if not os.path.isfile(archive_path):\n result = self.site.needFile(inner_path, priority=10)\n self.site.updateWebsocket(file_done=inner_path)\n if not result:\n raise Exception(\"Unable to download file\")\n file_obj = self.site.storage.openBigfile(inner_path)\n if file_obj == False:\n file_obj = None\n\n try:\n archive = openArchive(archive_path, file_obj=file_obj)\n except Exception as err:\n raise Exception(\"Unable to download file: %s\" % Debug.formatException(err))\n\n return archive\n\n def walk(self, inner_path, *args, **kwags):\n if \".zip\" in inner_path or \".tar.gz\" in inner_path:\n match = re.match(\"^(.*\\.(?:tar.gz|zip))(.*)\", inner_path)\n archive_inner_path, path_within = match.groups()\n archive = self.openArchive(archive_inner_path)\n path_within = path_within.lstrip(\"/\")\n\n if archive_inner_path.endswith(\".zip\"):\n namelist = [name for name in archive.namelist() if not name.endswith(\"/\")]\n else:\n namelist = [item.name for item in archive.getmembers() if not item.isdir()]\n\n namelist_relative = []\n for name in namelist:\n if not name.startswith(path_within):\n continue\n name_relative = name.replace(path_within, \"\", 1).rstrip(\"/\")\n namelist_relative.append(name_relative)\n\n return namelist_relative\n\n else:\n return super(SiteStoragePlugin, self).walk(inner_path, *args, **kwags)\n\n def list(self, inner_path, *args, **kwags):\n if \".zip\" in inner_path or \".tar.gz\" in inner_path:\n match = re.match(\"^(.*\\.(?:tar.gz|zip))(.*)\", inner_path)\n archive_inner_path, path_within = match.groups()\n archive = self.openArchive(archive_inner_path)\n path_within = path_within.lstrip(\"/\")\n\n if archive_inner_path.endswith(\".zip\"):\n namelist = [name for name in archive.namelist()]\n else:\n namelist = [item.name for item in archive.getmembers()]\n\n namelist_relative = []\n for name in namelist:\n if not name.startswith(path_within):\n continue\n name_relative = name.replace(path_within, \"\", 1).rstrip(\"/\")\n\n if \"/\" in name_relative: # File is in sub-directory\n continue\n\n namelist_relative.append(name_relative)\n return namelist_relative\n\n else:\n return super(SiteStoragePlugin, self).list(inner_path, *args, **kwags)\n\n def read(self, inner_path, mode=\"rb\", **kwargs):\n if \".zip/\" in inner_path or \".tar.gz/\" in inner_path:\n match = re.match(\"^(.*\\.(?:tar.gz|zip))(.*)\", inner_path)\n archive_inner_path, path_within = match.groups()\n archive = self.openArchive(archive_inner_path)\n path_within = path_within.lstrip(\"/\")\n\n if archive_inner_path.endswith(\".zip\"):\n return archive.open(path_within).read()\n else:\n return archive.extractfile(path_within).read()\n\n else:\n return super(SiteStoragePlugin, self).read(inner_path, mode, **kwargs)\n\n"
},
{
"path": "plugins/FilePack/__init__.py",
"content": "from . import FilePackPlugin"
},
{
"path": "plugins/FilePack/plugin_info.json",
"content": "{\n\t\"name\": \"FilePack\",\n\t\"description\": \"Transparent web access for Zip and Tar.gz files.\",\n\t\"default\": \"enabled\"\n}"
},
{
"path": "plugins/MergerSite/MergerSitePlugin.py",
"content": "import re\nimport time\nimport copy\nimport os\n\nfrom Plugin import PluginManager\nfrom Translate import Translate\nfrom util import RateLimit\nfrom util import helper\nfrom util.Flag import flag\nfrom Debug import Debug\ntry:\n import OptionalManager.UiWebsocketPlugin # To make optioanlFileInfo merger sites compatible\nexcept Exception:\n pass\n\nif \"merger_db\" not in locals().keys(): # To keep merger_sites between module reloads\n merger_db = {} # Sites that allowed to list other sites {address: [type1, type2...]}\n merged_db = {} # Sites that allowed to be merged to other sites {address: type, ...}\n merged_to_merger = {} # {address: [site1, site2, ...]} cache\n site_manager = None # Site manager for merger sites\n\n\nplugin_dir = os.path.dirname(__file__)\n\nif \"_\" not in locals():\n _ = Translate(plugin_dir + \"/languages/\")\n\n\n# Check if the site has permission to this merger site\ndef checkMergerPath(address, inner_path):\n merged_match = re.match(\"^merged-(.*?)/([A-Za-z0-9]{26,35})/\", inner_path)\n if merged_match:\n merger_type = merged_match.group(1)\n # Check if merged site is allowed to include other sites\n if merger_type in merger_db.get(address, []):\n # Check if included site allows to include\n merged_address = merged_match.group(2)\n if merged_db.get(merged_address) == merger_type:\n inner_path = re.sub(\"^merged-(.*?)/([A-Za-z0-9]{26,35})/\", \"\", inner_path)\n return merged_address, inner_path\n else:\n raise Exception(\n \"Merger site (%s) does not have permission for merged site: %s (%s)\" %\n (merger_type, merged_address, merged_db.get(merged_address))\n )\n else:\n raise Exception(\"No merger (%s) permission to load: Site blocked
\nThis site is on your blocklist:
\n\n
\n \nToo much image
\n on 2015-01-25 12:32:11
\n %s (%s not in %s)\" % (\n address, inner_path, merger_type, merger_db.get(address, []))\n )\n else:\n raise Exception(\"Invalid merger path: %s\" % inner_path)\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n # Download new site\n def actionMergerSiteAdd(self, to, addresses):\n if type(addresses) != list:\n # Single site add\n addresses = [addresses]\n # Check if the site has merger permission\n merger_types = merger_db.get(self.site.address)\n if not merger_types:\n return self.response(to, {\"error\": \"Not a merger site\"})\n\n if RateLimit.isAllowed(self.site.address + \"-MergerSiteAdd\", 10) and len(addresses) == 1:\n # Without confirmation if only one site address and not called in last 10 sec\n self.cbMergerSiteAdd(to, addresses)\n else:\n self.cmd(\n \"confirm\",\n [_[\"Add %s new site?\"] % len(addresses), \"Add\"],\n lambda res: self.cbMergerSiteAdd(to, addresses)\n )\n self.response(to, \"ok\")\n\n # Callback of adding new site confirmation\n def cbMergerSiteAdd(self, to, addresses):\n added = 0\n for address in addresses:\n try:\n site_manager.need(address)\n added += 1\n except Exception as err:\n self.cmd(\"notification\", [\"error\", _[\"Adding %s failed: %s\"] % (address, err)])\n if added:\n self.cmd(\"notification\", [\"done\", _[\"Added %s new site\"] % added, 5000])\n RateLimit.called(self.site.address + \"-MergerSiteAdd\")\n site_manager.updateMergerSites()\n\n # Delete a merged site\n @flag.no_multiuser\n def actionMergerSiteDelete(self, to, address):\n site = self.server.sites.get(address)\n if not site:\n return self.response(to, {\"error\": \"No site found: %s\" % address})\n\n merger_types = merger_db.get(self.site.address)\n if not merger_types:\n return self.response(to, {\"error\": \"Not a merger site\"})\n if merged_db.get(address) not in merger_types:\n return self.response(to, {\"error\": \"Merged type (%s) not in %s\" % (merged_db.get(address), merger_types)})\n\n self.cmd(\"notification\", [\"done\", _[\"Site deleted: %s\"] % address, 5000])\n self.response(to, \"ok\")\n\n # Lists merged sites\n def actionMergerSiteList(self, to, query_site_info=False):\n merger_types = merger_db.get(self.site.address)\n ret = {}\n if not merger_types:\n return self.response(to, {\"error\": \"Not a merger site\"})\n for address, merged_type in merged_db.items():\n if merged_type not in merger_types:\n continue # Site not for us\n if query_site_info:\n site = self.server.sites.get(address)\n ret[address] = self.formatSiteInfo(site, create_user=False)\n else:\n ret[address] = merged_type\n self.response(to, ret)\n\n def hasSitePermission(self, address, *args, **kwargs):\n if super(UiWebsocketPlugin, self).hasSitePermission(address, *args, **kwargs):\n return True\n else:\n if self.site.address in [merger_site.address for merger_site in merged_to_merger.get(address, [])]:\n return True\n else:\n return False\n\n # Add support merger sites for file commands\n def mergerFuncWrapper(self, func_name, to, inner_path, *args, **kwargs):\n if inner_path.startswith(\"merged-\"):\n merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)\n\n # Set the same cert for merged site\n merger_cert = self.user.getSiteData(self.site.address).get(\"cert\")\n if merger_cert and self.user.getSiteData(merged_address).get(\"cert\") != merger_cert:\n self.user.setCert(merged_address, merger_cert)\n\n req_self = copy.copy(self)\n req_self.site = self.server.sites.get(merged_address) # Change the site to the merged one\n\n func = getattr(super(UiWebsocketPlugin, req_self), func_name)\n return func(to, merged_inner_path, *args, **kwargs)\n else:\n func = getattr(super(UiWebsocketPlugin, self), func_name)\n return func(to, inner_path, *args, **kwargs)\n\n def actionFileList(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileList\", to, inner_path, *args, **kwargs)\n\n def actionDirList(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionDirList\", to, inner_path, *args, **kwargs)\n\n def actionFileGet(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileGet\", to, inner_path, *args, **kwargs)\n\n def actionFileWrite(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileWrite\", to, inner_path, *args, **kwargs)\n\n def actionFileDelete(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileDelete\", to, inner_path, *args, **kwargs)\n\n def actionFileRules(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileRules\", to, inner_path, *args, **kwargs)\n\n def actionFileNeed(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionFileNeed\", to, inner_path, *args, **kwargs)\n\n def actionOptionalFileInfo(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionOptionalFileInfo\", to, inner_path, *args, **kwargs)\n\n def actionOptionalFileDelete(self, to, inner_path, *args, **kwargs):\n return self.mergerFuncWrapper(\"actionOptionalFileDelete\", to, inner_path, *args, **kwargs)\n\n def actionBigfileUploadInit(self, to, inner_path, *args, **kwargs):\n back = self.mergerFuncWrapper(\"actionBigfileUploadInit\", to, inner_path, *args, **kwargs)\n if inner_path.startswith(\"merged-\"):\n merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)\n back[\"inner_path\"] = \"merged-%s/%s/%s\" % (merged_db[merged_address], merged_address, back[\"inner_path\"])\n return back\n\n # Add support merger sites for file commands with privatekey parameter\n def mergerFuncWrapperWithPrivatekey(self, func_name, to, privatekey, inner_path, *args, **kwargs):\n func = getattr(super(UiWebsocketPlugin, self), func_name)\n if inner_path.startswith(\"merged-\"):\n merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)\n merged_site = self.server.sites.get(merged_address)\n\n # Set the same cert for merged site\n merger_cert = self.user.getSiteData(self.site.address).get(\"cert\")\n if merger_cert:\n self.user.setCert(merged_address, merger_cert)\n\n site_before = self.site # Save to be able to change it back after we ran the command\n self.site = merged_site # Change the site to the merged one\n try:\n back = func(to, privatekey, merged_inner_path, *args, **kwargs)\n finally:\n self.site = site_before # Change back to original site\n return back\n else:\n return func(to, privatekey, inner_path, *args, **kwargs)\n\n def actionSiteSign(self, to, privatekey=None, inner_path=\"content.json\", *args, **kwargs):\n return self.mergerFuncWrapperWithPrivatekey(\"actionSiteSign\", to, privatekey, inner_path, *args, **kwargs)\n\n def actionSitePublish(self, to, privatekey=None, inner_path=\"content.json\", *args, **kwargs):\n return self.mergerFuncWrapperWithPrivatekey(\"actionSitePublish\", to, privatekey, inner_path, *args, **kwargs)\n\n def actionPermissionAdd(self, to, permission):\n super(UiWebsocketPlugin, self).actionPermissionAdd(to, permission)\n if permission.startswith(\"Merger\"):\n self.site.storage.rebuildDb()\n\n def actionPermissionDetails(self, to, permission):\n if not permission.startswith(\"Merger\"):\n return super(UiWebsocketPlugin, self).actionPermissionDetails(to, permission)\n\n merger_type = permission.replace(\"Merger:\", \"\")\n if not re.match(\"^[A-Za-z0-9-]+$\", merger_type):\n raise Exception(\"Invalid merger_type: %s\" % merger_type)\n merged_sites = []\n for address, merged_type in merged_db.items():\n if merged_type != merger_type:\n continue\n site = self.server.sites.get(address)\n try:\n merged_sites.append(site.content_manager.contents.get(\"content.json\").get(\"title\", address))\n except Exception:\n merged_sites.append(address)\n\n details = _[\"Read and write permissions to sites with merged type of %s \"] % merger_type\n details += _[\"(%s sites)\"] % len(merged_sites)\n details += \"
%s
\" % \", \".join(merged_sites)\n self.response(to, details)\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n # Allow to load merged site files using /merged-ZeroMe/address/file.jpg\n def parsePath(self, path):\n path_parts = super(UiRequestPlugin, self).parsePath(path)\n if \"merged-\" not in path: # Optimization\n return path_parts\n path_parts[\"address\"], path_parts[\"inner_path\"] = checkMergerPath(path_parts[\"address\"], path_parts[\"inner_path\"])\n return path_parts\n\n\n@PluginManager.registerTo(\"SiteStorage\")\nclass SiteStoragePlugin(object):\n # Also rebuild from merged sites\n def getDbFiles(self):\n merger_types = merger_db.get(self.site.address)\n\n # First return the site's own db files\n for item in super(SiteStoragePlugin, self).getDbFiles():\n yield item\n\n # Not a merger site, that's all\n if not merger_types:\n return\n\n merged_sites = [\n site_manager.sites[address]\n for address, merged_type in merged_db.items()\n if merged_type in merger_types\n ]\n found = 0\n for merged_site in merged_sites:\n self.log.debug(\"Loading merged site: %s\" % merged_site)\n merged_type = merged_db[merged_site.address]\n for content_inner_path, content in merged_site.content_manager.contents.items():\n # content.json file itself\n if merged_site.storage.isFile(content_inner_path): # Missing content.json file\n merged_inner_path = \"merged-%s/%s/%s\" % (merged_type, merged_site.address, content_inner_path)\n yield merged_inner_path, merged_site.storage.getPath(content_inner_path)\n else:\n merged_site.log.error(\"[MISSING] %s\" % content_inner_path)\n # Data files in content.json\n content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site\n for file_relative_path in list(content.get(\"files\", {}).keys()) + list(content.get(\"files_optional\", {}).keys()):\n if not file_relative_path.endswith(\".json\"):\n continue # We only interesed in json files\n file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir\n file_inner_path = file_inner_path.strip(\"/\") # Strip leading /\n if merged_site.storage.isFile(file_inner_path):\n merged_inner_path = \"merged-%s/%s/%s\" % (merged_type, merged_site.address, file_inner_path)\n yield merged_inner_path, merged_site.storage.getPath(file_inner_path)\n else:\n merged_site.log.error(\"[MISSING] %s\" % file_inner_path)\n found += 1\n if found % 100 == 0:\n time.sleep(0.001) # Context switch to avoid UI block\n\n # Also notice merger sites on a merged site file change\n def onUpdated(self, inner_path, file=None):\n super(SiteStoragePlugin, self).onUpdated(inner_path, file)\n\n merged_type = merged_db.get(self.site.address)\n\n for merger_site in merged_to_merger.get(self.site.address, []):\n if merger_site.address == self.site.address: # Avoid infinite loop\n continue\n virtual_path = \"merged-%s/%s/%s\" % (merged_type, self.site.address, inner_path)\n if inner_path.endswith(\".json\"):\n if file is not None:\n merger_site.storage.onUpdated(virtual_path, file=file)\n else:\n merger_site.storage.onUpdated(virtual_path, file=self.open(inner_path))\n else:\n merger_site.storage.onUpdated(virtual_path)\n\n\n@PluginManager.registerTo(\"Site\")\nclass SitePlugin(object):\n def fileDone(self, inner_path):\n super(SitePlugin, self).fileDone(inner_path)\n\n for merger_site in merged_to_merger.get(self.address, []):\n if merger_site.address == self.address:\n continue\n for ws in merger_site.websockets:\n ws.event(\"siteChanged\", self, {\"event\": [\"file_done\", inner_path]})\n\n def fileFailed(self, inner_path):\n super(SitePlugin, self).fileFailed(inner_path)\n\n for merger_site in merged_to_merger.get(self.address, []):\n if merger_site.address == self.address:\n continue\n for ws in merger_site.websockets:\n ws.event(\"siteChanged\", self, {\"event\": [\"file_failed\", inner_path]})\n\n\n@PluginManager.registerTo(\"SiteManager\")\nclass SiteManagerPlugin(object):\n # Update merger site for site types\n def updateMergerSites(self):\n global merger_db, merged_db, merged_to_merger, site_manager\n s = time.time()\n merger_db_new = {}\n merged_db_new = {}\n merged_to_merger_new = {}\n site_manager = self\n if not self.sites:\n return\n for site in self.sites.values():\n # Update merged sites\n try:\n merged_type = site.content_manager.contents.get(\"content.json\", {}).get(\"merged_type\")\n except Exception as err:\n self.log.error(\"Error loading site %s: %s\" % (site.address, Debug.formatException(err)))\n continue\n if merged_type:\n merged_db_new[site.address] = merged_type\n\n # Update merger sites\n for permission in site.settings[\"permissions\"]:\n if not permission.startswith(\"Merger:\"):\n continue\n if merged_type:\n self.log.error(\n \"Removing permission %s from %s: Merger and merged at the same time.\" %\n (permission, site.address)\n )\n site.settings[\"permissions\"].remove(permission)\n continue\n merger_type = permission.replace(\"Merger:\", \"\")\n if site.address not in merger_db_new:\n merger_db_new[site.address] = []\n merger_db_new[site.address].append(merger_type)\n site_manager.sites[site.address] = site\n\n # Update merged to merger\n if merged_type:\n for merger_site in self.sites.values():\n if \"Merger:\" + merged_type in merger_site.settings[\"permissions\"]:\n if site.address not in merged_to_merger_new:\n merged_to_merger_new[site.address] = []\n merged_to_merger_new[site.address].append(merger_site)\n\n # Update globals\n merger_db = merger_db_new\n merged_db = merged_db_new\n merged_to_merger = merged_to_merger_new\n\n self.log.debug(\"Updated merger sites in %.3fs\" % (time.time() - s))\n\n def load(self, *args, **kwags):\n super(SiteManagerPlugin, self).load(*args, **kwags)\n self.updateMergerSites()\n\n def saveDelayed(self, *args, **kwags):\n super(SiteManagerPlugin, self).saveDelayed(*args, **kwags)\n self.updateMergerSites()\n"
},
{
"path": "plugins/MergerSite/__init__.py",
"content": "from . import MergerSitePlugin"
},
{
"path": "plugins/MergerSite/languages/es.json",
"content": "{\n\t\"Add %s new site?\": \"¿Agregar %s nuevo sitio?\",\n\t\"Added %s new site\": \"Sitio %s agregado\",\n\t\"Site deleted: %s\": \"Sitio removido: %s\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/fr.json",
"content": "{\n\t\"Add %s new site?\": \"Ajouter le site %s ?\",\n\t\"Added %s new site\": \"Site %s ajouté\",\n\t\"Site deleted: %s\": \"Site %s supprimé\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/hu.json",
"content": "{\n\t\"Add %s new site?\": \"Új oldal hozzáadása: %s?\",\n\t\"Added %s new site\": \"Új oldal hozzáadva: %s\",\n\t\"Site deleted: %s\": \"Oldal törölve: %s\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/it.json",
"content": "{\n\t\"Add %s new site?\": \"Aggiungere %s nuovo sito ?\",\n\t\"Added %s new site\": \"Sito %s aggiunto\",\n\t\"Site deleted: %s\": \"Sito %s eliminato\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/jp.json",
"content": "{\n\t\"Add %s new site?\": \"サイト: %s を追加しますか？\",\n\t\"Added %s new site\": \"サイト: %s を追加しました\",\n\t\"Site deleted: %s\": \"サイト: %s を削除しました\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/pt-br.json",
"content": "{\n\t\"Add %s new site?\": \"Adicionar %s novo site?\",\n\t\"Added %s new site\": \"Site %s adicionado\",\n\t\"Site deleted: %s\": \"Site removido: %s\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/tr.json",
"content": "{\n\t\"Add %s new site?\": \"%s sitesi eklensin mi?\",\n\t\"Added %s new site\": \"%s sitesi eklendi\",\n\t\"Site deleted: %s\": \"%s sitesi silindi\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/zh-tw.json",
"content": "{\n\t\"Add %s new site?\": \"添加新網站: %s？\",\n\t\"Added %s new site\": \"已添加到新網站：%s\",\n\t\"Site deleted: %s\": \"網站已刪除：%s\"\n}\n"
},
{
"path": "plugins/MergerSite/languages/zh.json",
"content": "{\n\t\"Add %s new site?\": \"添加新站点: %s？\",\n\t\"Added %s new site\": \"已添加到新站点：%s\",\n\t\"Site deleted: %s\": \"站点已删除：%s\"\n}\n"
},
{
"path": "plugins/Newsfeed/NewsfeedPlugin.py",
"content": "import time\nimport re\n\nfrom Plugin import PluginManager\nfrom Db.DbQuery import DbQuery\nfrom Debug import Debug\nfrom util import helper\nfrom util.Flag import flag\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def formatSiteInfo(self, site, create_user=True):\n site_info = super(UiWebsocketPlugin, self).formatSiteInfo(site, create_user=create_user)\n feed_following = self.user.sites.get(site.address, {}).get(\"follow\", None)\n if feed_following == None:\n site_info[\"feed_follow_num\"] = None\n else:\n site_info[\"feed_follow_num\"] = len(feed_following)\n return site_info\n\n def actionFeedFollow(self, to, feeds):\n self.user.setFeedFollow(self.site.address, feeds)\n self.user.save()\n self.response(to, \"ok\")\n\n def actionFeedListFollow(self, to):\n feeds = self.user.sites.get(self.site.address, {}).get(\"follow\", {})\n self.response(to, feeds)\n\n @flag.admin\n def actionFeedQuery(self, to, limit=10, day_limit=3):\n from Site import SiteManager\n rows = []\n stats = []\n\n total_s = time.time()\n num_sites = 0\n\n for address, site_data in list(self.user.sites.items()):\n feeds = site_data.get(\"follow\")\n if not feeds:\n continue\n if type(feeds) is not dict:\n self.log.debug(\"Invalid feed for site %s\" % address)\n continue\n num_sites += 1\n for name, query_set in feeds.items():\n site = SiteManager.site_manager.get(address)\n if not site or not site.storage.has_db:\n continue\n\n s = time.time()\n try:\n query_raw, params = query_set\n query_parts = re.split(r\"UNION(?:\\s+ALL|)\", query_raw)\n for i, query_part in enumerate(query_parts):\n db_query = DbQuery(query_part)\n if day_limit:\n where = \" WHERE %s > strftime('%%s', 'now', '-%s day')\" % (db_query.fields.get(\"date_added\", \"date_added\"), day_limit)\n if \"WHERE\" in query_part:\n query_part = re.sub(\"WHERE (.*?)(?=$| GROUP BY)\", where+\" AND (\\\\1)\", query_part)\n else:\n query_part += where\n query_parts[i] = query_part\n query = \" UNION \".join(query_parts)\n\n if \":params\" in query:\n query_params = map(helper.sqlquote, params)\n query = query.replace(\":params\", \",\".join(query_params))\n\n res = site.storage.query(query + \" ORDER BY date_added DESC LIMIT %s\" % limit)\n\n except Exception as err: # Log error\n self.log.error(\"%s feed query %s error: %s\" % (address, name, Debug.formatException(err)))\n stats.append({\"site\": site.address, \"feed_name\": name, \"error\": str(err)})\n continue\n\n for row in res:\n row = dict(row)\n if not isinstance(row[\"date_added\"], (int, float, complex)):\n self.log.debug(\"Invalid date_added from site %s: %r\" % (address, row[\"date_added\"]))\n continue\n if row[\"date_added\"] > 1000000000000: # Formatted as millseconds\n row[\"date_added\"] = row[\"date_added\"] / 1000\n if \"date_added\" not in row or row[\"date_added\"] > time.time() + 120:\n self.log.debug(\"Newsfeed item from the future from from site %s\" % address)\n continue # Feed item is in the future, skip it\n row[\"site\"] = address\n row[\"feed_name\"] = name\n rows.append(row)\n stats.append({\"site\": site.address, \"feed_name\": name, \"taken\": round(time.time() - s, 3)})\n time.sleep(0.001)\n return self.response(to, {\"rows\": rows, \"stats\": stats, \"num\": len(rows), \"sites\": num_sites, \"taken\": round(time.time() - total_s, 3)})\n\n def parseSearch(self, search):\n parts = re.split(\"(site|type):\", search)\n if len(parts) > 1: # Found filter\n search_text = parts[0]\n parts = [part.strip() for part in parts]\n filters = dict(zip(parts[1::2], parts[2::2]))\n else:\n search_text = search\n filters = {}\n return [search_text, filters]\n\n def actionFeedSearch(self, to, search, limit=30, day_limit=30):\n if \"ADMIN\" not in self.site.settings[\"permissions\"]:\n return self.response(to, \"FeedSearch not allowed\")\n\n from Site import SiteManager\n rows = []\n stats = []\n num_sites = 0\n total_s = time.time()\n\n search_text, filters = self.parseSearch(search)\n\n for address, site in SiteManager.site_manager.list().items():\n if not site.storage.has_db:\n continue\n\n if \"site\" in filters:\n if filters[\"site\"].lower() not in [site.address, site.content_manager.contents[\"content.json\"].get(\"title\").lower()]:\n continue\n\n if site.storage.db: # Database loaded\n feeds = site.storage.db.schema.get(\"feeds\")\n else:\n try:\n feeds = site.storage.loadJson(\"dbschema.json\").get(\"feeds\")\n except:\n continue\n\n if not feeds:\n continue\n\n num_sites += 1\n\n for name, query in feeds.items():\n s = time.time()\n try:\n db_query = DbQuery(query)\n\n params = []\n # Filters\n if search_text:\n db_query.wheres.append(\"(%s LIKE ? OR %s LIKE ?)\" % (db_query.fields[\"body\"], db_query.fields[\"title\"]))\n search_like = \"%\" + search_text.replace(\" \", \"%\") + \"%\"\n params.append(search_like)\n params.append(search_like)\n if filters.get(\"type\") and filters[\"type\"] not in query:\n continue\n\n if day_limit:\n db_query.wheres.append(\n \"%s > strftime('%%s', 'now', '-%s day')\" % (db_query.fields.get(\"date_added\", \"date_added\"), day_limit)\n )\n\n # Order\n db_query.parts[\"ORDER BY\"] = \"date_added DESC\"\n db_query.parts[\"LIMIT\"] = str(limit)\n\n res = site.storage.query(str(db_query), params)\n except Exception as err:\n self.log.error(\"%s feed query %s error: %s\" % (address, name, Debug.formatException(err)))\n stats.append({\"site\": site.address, \"feed_name\": name, \"error\": str(err), \"query\": query})\n continue\n for row in res:\n row = dict(row)\n if not row[\"date_added\"] or row[\"date_added\"] > time.time() + 120:\n continue # Feed item is in the future, skip it\n row[\"site\"] = address\n row[\"feed_name\"] = name\n rows.append(row)\n stats.append({\"site\": site.address, \"feed_name\": name, \"taken\": round(time.time() - s, 3)})\n return self.response(to, {\"rows\": rows, \"num\": len(rows), \"sites\": num_sites, \"taken\": round(time.time() - total_s, 3), \"stats\": stats})\n\n\n@PluginManager.registerTo(\"User\")\nclass UserPlugin(object):\n # Set queries that user follows\n def setFeedFollow(self, address, feeds):\n site_data = self.getSiteData(address)\n site_data[\"follow\"] = feeds\n self.save()\n return site_data\n"
},
{
"path": "plugins/Newsfeed/__init__.py",
"content": "from . import NewsfeedPlugin"
},
{
"path": "plugins/OptionalManager/ContentDbPlugin.py",
"content": "import time\nimport collections\nimport itertools\nimport re\n\nimport gevent\n\nfrom util import helper\nfrom Plugin import PluginManager\nfrom Config import config\nfrom Debug import Debug\n\nif \"content_db\" not in locals().keys(): # To keep between module reloads\n content_db = None\n\n\n@PluginManager.registerTo(\"ContentDb\")\nclass ContentDbPlugin(object):\n def __init__(self, *args, **kwargs):\n global content_db\n content_db = self\n self.filled = {} # Site addresses that already filled from content.json\n self.need_filling = False # file_optional table just created, fill data from content.json files\n self.time_peer_numbers_updated = 0\n self.my_optional_files = {} # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files)\n self.optional_files = collections.defaultdict(dict)\n self.optional_files_loaded = False\n self.timer_check_optional = helper.timer(60 * 5, self.checkOptionalLimit)\n super(ContentDbPlugin, self).__init__(*args, **kwargs)\n\n def getSchema(self):\n schema = super(ContentDbPlugin, self).getSchema()\n\n # Need file_optional table\n schema[\"tables\"][\"file_optional\"] = {\n \"cols\": [\n [\"file_id\", \"INTEGER PRIMARY KEY UNIQUE NOT NULL\"],\n [\"site_id\", \"INTEGER REFERENCES site (site_id) ON DELETE CASCADE\"],\n [\"inner_path\", \"TEXT\"],\n [\"hash_id\", \"INTEGER\"],\n [\"size\", \"INTEGER\"],\n [\"peer\", \"INTEGER DEFAULT 0\"],\n [\"uploaded\", \"INTEGER DEFAULT 0\"],\n [\"is_downloaded\", \"INTEGER DEFAULT 0\"],\n [\"is_pinned\", \"INTEGER DEFAULT 0\"],\n [\"time_added\", \"INTEGER DEFAULT 0\"],\n [\"time_downloaded\", \"INTEGER DEFAULT 0\"],\n [\"time_accessed\", \"INTEGER DEFAULT 0\"]\n ],\n \"indexes\": [\n \"CREATE UNIQUE INDEX file_optional_key ON file_optional (site_id, inner_path)\",\n \"CREATE INDEX is_downloaded ON file_optional (is_downloaded)\"\n ],\n \"schema_changed\": 11\n }\n\n return schema\n\n def initSite(self, site):\n super(ContentDbPlugin, self).initSite(site)\n if self.need_filling:\n self.fillTableFileOptional(site)\n\n def checkTables(self):\n changed_tables = super(ContentDbPlugin, self).checkTables()\n if \"file_optional\" in changed_tables:\n self.need_filling = True\n return changed_tables\n\n # Load optional files ending\n def loadFilesOptional(self):\n s = time.time()\n num = 0\n total = 0\n total_downloaded = 0\n res = content_db.execute(\"SELECT site_id, inner_path, size, is_downloaded FROM file_optional\")\n site_sizes = collections.defaultdict(lambda: collections.defaultdict(int))\n for row in res:\n self.optional_files[row[\"site_id\"]][row[\"inner_path\"][-8:]] = 1\n num += 1\n\n # Update site size stats\n site_sizes[row[\"site_id\"]][\"size_optional\"] += row[\"size\"]\n if row[\"is_downloaded\"]:\n site_sizes[row[\"site_id\"]][\"optional_downloaded\"] += row[\"size\"]\n\n # Site site size stats to sites.json settings\n site_ids_reverse = {val: key for key, val in self.site_ids.items()}\n for site_id, stats in site_sizes.items():\n site_address = site_ids_reverse.get(site_id)\n if not site_address or site_address not in self.sites:\n self.log.error(\"Not found site_id: %s\" % site_id)\n continue\n site = self.sites[site_address]\n site.settings[\"size_optional\"] = stats[\"size_optional\"]\n site.settings[\"optional_downloaded\"] = stats[\"optional_downloaded\"]\n total += stats[\"size_optional\"]\n total_downloaded += stats[\"optional_downloaded\"]\n\n self.log.info(\n \"Loaded %s optional files: %.2fMB, downloaded: %.2fMB in %.3fs\" %\n (num, float(total) / 1024 / 1024, float(total_downloaded) / 1024 / 1024, time.time() - s)\n )\n\n if self.need_filling and self.getOptionalLimitBytes() >= 0 and self.getOptionalLimitBytes() < total_downloaded:\n limit_bytes = self.getOptionalLimitBytes()\n limit_new = round((float(total_downloaded) / 1024 / 1024 / 1024) * 1.1, 2) # Current limit + 10%\n self.log.info(\n \"First startup after update and limit is smaller than downloaded files size (%.2fGB), increasing it from %.2fGB to %.2fGB\" %\n (float(total_downloaded) / 1024 / 1024 / 1024, float(limit_bytes) / 1024 / 1024 / 1024, limit_new)\n )\n config.saveValue(\"optional_limit\", limit_new)\n config.optional_limit = str(limit_new)\n\n # Predicts if the file is optional\n def isOptionalFile(self, site_id, inner_path):\n return self.optional_files[site_id].get(inner_path[-8:])\n\n # Fill file_optional table with optional files found in sites\n def fillTableFileOptional(self, site):\n s = time.time()\n site_id = self.site_ids.get(site.address)\n if not site_id:\n return False\n cur = self.getCursor()\n res = cur.execute(\"SELECT * FROM content WHERE size_files_optional > 0 AND site_id = %s\" % site_id)\n num = 0\n for row in res.fetchall():\n content = site.content_manager.contents[row[\"inner_path\"]]\n try:\n num += self.setContentFilesOptional(site, row[\"inner_path\"], content, cur=cur)\n except Exception as err:\n self.log.error(\"Error loading %s into file_optional: %s\" % (row[\"inner_path\"], err))\n cur.close()\n\n # Set my files to pinned\n from User import UserManager\n user = UserManager.user_manager.get()\n if not user:\n user = UserManager.user_manager.create()\n auth_address = user.getAuthAddress(site.address)\n res = self.execute(\n \"UPDATE file_optional SET is_pinned = 1 WHERE site_id = :site_id AND inner_path LIKE :inner_path\",\n {\"site_id\": site_id, \"inner_path\": \"%%/%s/%%\" % auth_address}\n )\n\n self.log.debug(\n \"Filled file_optional table for %s in %.3fs (loaded: %s, is_pinned: %s)\" %\n (site.address, time.time() - s, num, res.rowcount)\n )\n self.filled[site.address] = True\n\n def setContentFilesOptional(self, site, content_inner_path, content, cur=None):\n if not cur:\n cur = self\n\n num = 0\n site_id = self.site_ids[site.address]\n content_inner_dir = helper.getDirname(content_inner_path)\n for relative_inner_path, file in content.get(\"files_optional\", {}).items():\n file_inner_path = content_inner_dir + relative_inner_path\n hash_id = int(file[\"sha512\"][0:4], 16)\n if hash_id in site.content_manager.hashfield:\n is_downloaded = 1\n else:\n is_downloaded = 0\n if site.address + \"/\" + content_inner_dir in self.my_optional_files:\n is_pinned = 1\n else:\n is_pinned = 0\n cur.insertOrUpdate(\"file_optional\", {\n \"hash_id\": hash_id,\n \"size\": int(file[\"size\"])\n }, {\n \"site_id\": site_id,\n \"inner_path\": file_inner_path\n }, oninsert={\n \"time_added\": int(time.time()),\n \"time_downloaded\": int(time.time()) if is_downloaded else 0,\n \"is_downloaded\": is_downloaded,\n \"peer\": is_downloaded,\n \"is_pinned\": is_pinned\n })\n self.optional_files[site_id][file_inner_path[-8:]] = 1\n num += 1\n\n return num\n\n def setContent(self, site, inner_path, content, size=0):\n super(ContentDbPlugin, self).setContent(site, inner_path, content, size=size)\n old_content = site.content_manager.contents.get(inner_path, {})\n if (not self.need_filling or self.filled.get(site.address)) and (\"files_optional\" in content or \"files_optional\" in old_content):\n self.setContentFilesOptional(site, inner_path, content)\n # Check deleted files\n if old_content:\n old_files = old_content.get(\"files_optional\", {}).keys()\n new_files = content.get(\"files_optional\", {}).keys()\n content_inner_dir = helper.getDirname(inner_path)\n deleted = [content_inner_dir + key for key in old_files if key not in new_files]\n if deleted:\n site_id = self.site_ids[site.address]\n self.execute(\"DELETE FROM file_optional WHERE ?\", {\"site_id\": site_id, \"inner_path\": deleted})\n\n def deleteContent(self, site, inner_path):\n content = site.content_manager.contents.get(inner_path)\n if content and \"files_optional\" in content:\n site_id = self.site_ids[site.address]\n content_inner_dir = helper.getDirname(inner_path)\n optional_inner_paths = [\n content_inner_dir + relative_inner_path\n for relative_inner_path in content.get(\"files_optional\", {}).keys()\n ]\n self.execute(\"DELETE FROM file_optional WHERE ?\", {\"site_id\": site_id, \"inner_path\": optional_inner_paths})\n super(ContentDbPlugin, self).deleteContent(site, inner_path)\n\n def updatePeerNumbers(self):\n s = time.time()\n num_file = 0\n num_updated = 0\n num_site = 0\n for site in list(self.sites.values()):\n if not site.content_manager.has_optional_files:\n continue\n if not site.isServing():\n continue\n has_updated_hashfield = next((\n peer\n for peer in site.peers.values()\n if peer.has_hashfield and peer.hashfield.time_changed > self.time_peer_numbers_updated\n ), None)\n\n if not has_updated_hashfield and site.content_manager.hashfield.time_changed < self.time_peer_numbers_updated:\n continue\n\n hashfield_peers = itertools.chain.from_iterable(\n peer.hashfield.storage\n for peer in site.peers.values()\n if peer.has_hashfield\n )\n peer_nums = collections.Counter(\n itertools.chain(\n hashfield_peers,\n site.content_manager.hashfield\n )\n )\n\n site_id = self.site_ids[site.address]\n if not site_id:\n continue\n\n res = self.execute(\"SELECT file_id, hash_id, peer FROM file_optional WHERE ?\", {\"site_id\": site_id})\n updates = {}\n for row in res:\n peer_num = peer_nums.get(row[\"hash_id\"], 0)\n if peer_num != row[\"peer\"]:\n updates[row[\"file_id\"]] = peer_num\n\n for file_id, peer_num in updates.items():\n self.execute(\"UPDATE file_optional SET peer = ? WHERE file_id = ?\", (peer_num, file_id))\n\n num_updated += len(updates)\n num_file += len(peer_nums)\n num_site += 1\n\n self.time_peer_numbers_updated = time.time()\n self.log.debug(\"%s/%s peer number for %s site updated in %.3fs\" % (num_updated, num_file, num_site, time.time() - s))\n\n def queryDeletableFiles(self):\n # First return the files with atleast 10 seeder and not accessed in last week\n query = \"\"\"\n SELECT * FROM file_optional\n WHERE peer > 10 AND %s\n ORDER BY time_accessed < %s DESC, uploaded / size\n \"\"\" % (self.getOptionalUsedWhere(), int(time.time() - 60 * 60 * 7))\n limit_start = 0\n while 1:\n num = 0\n res = self.execute(\"%s LIMIT %s, 50\" % (query, limit_start))\n for row in res:\n yield row\n num += 1\n if num < 50:\n break\n limit_start += 50\n\n self.log.debug(\"queryDeletableFiles returning less-seeded files\")\n\n # Then return files less seeder but still not accessed in last week\n query = \"\"\"\n SELECT * FROM file_optional\n WHERE peer <= 10 AND %s\n ORDER BY peer DESC, time_accessed < %s DESC, uploaded / size\n \"\"\" % (self.getOptionalUsedWhere(), int(time.time() - 60 * 60 * 7))\n limit_start = 0\n while 1:\n num = 0\n res = self.execute(\"%s LIMIT %s, 50\" % (query, limit_start))\n for row in res:\n yield row\n num += 1\n if num < 50:\n break\n limit_start += 50\n\n self.log.debug(\"queryDeletableFiles returning everyting\")\n\n # At the end return all files\n query = \"\"\"\n SELECT * FROM file_optional\n WHERE peer <= 10 AND %s\n ORDER BY peer DESC, time_accessed, uploaded / size\n \"\"\" % self.getOptionalUsedWhere()\n limit_start = 0\n while 1:\n num = 0\n res = self.execute(\"%s LIMIT %s, 50\" % (query, limit_start))\n for row in res:\n yield row\n num += 1\n if num < 50:\n break\n limit_start += 50\n\n def getOptionalLimitBytes(self):\n if config.optional_limit.endswith(\"%\"):\n limit_percent = float(re.sub(\"[^0-9.]\", \"\", config.optional_limit))\n limit_bytes = helper.getFreeSpace() * (limit_percent / 100)\n else:\n limit_bytes = float(re.sub(\"[^0-9.]\", \"\", config.optional_limit)) * 1024 * 1024 * 1024\n return limit_bytes\n\n def getOptionalUsedWhere(self):\n maxsize = config.optional_limit_exclude_minsize * 1024 * 1024\n query = \"is_downloaded = 1 AND is_pinned = 0 AND size < %s\" % maxsize\n\n # Don't delete optional files from owned sites\n my_site_ids = []\n for address, site in self.sites.items():\n if site.settings[\"own\"]:\n my_site_ids.append(str(self.site_ids[address]))\n\n if my_site_ids:\n query += \" AND site_id NOT IN (%s)\" % \", \".join(my_site_ids)\n return query\n\n def getOptionalUsedBytes(self):\n size = self.execute(\"SELECT SUM(size) FROM file_optional WHERE %s\" % self.getOptionalUsedWhere()).fetchone()[0]\n if not size:\n size = 0\n return size\n\n def getOptionalNeedDelete(self, size):\n if config.optional_limit.endswith(\"%\"):\n limit_percent = float(re.sub(\"[^0-9.]\", \"\", config.optional_limit))\n need_delete = size - ((helper.getFreeSpace() + size) * (limit_percent / 100))\n else:\n need_delete = size - self.getOptionalLimitBytes()\n return need_delete\n\n def checkOptionalLimit(self, limit=None):\n if not limit:\n limit = self.getOptionalLimitBytes()\n\n if limit < 0:\n self.log.debug(\"Invalid limit for optional files: %s\" % limit)\n return False\n\n size = self.getOptionalUsedBytes()\n\n need_delete = self.getOptionalNeedDelete(size)\n\n self.log.debug(\n \"Optional size: %.1fMB/%.1fMB, Need delete: %.1fMB\" %\n (float(size) / 1024 / 1024, float(limit) / 1024 / 1024, float(need_delete) / 1024 / 1024)\n )\n if need_delete <= 0:\n return False\n\n self.updatePeerNumbers()\n\n site_ids_reverse = {val: key for key, val in self.site_ids.items()}\n deleted_file_ids = []\n for row in self.queryDeletableFiles():\n site_address = site_ids_reverse.get(row[\"site_id\"])\n site = self.sites.get(site_address)\n if not site:\n self.log.error(\"No site found for id: %s\" % row[\"site_id\"])\n continue\n site.log.debug(\"Deleting %s %.3f MB left\" % (row[\"inner_path\"], float(need_delete) / 1024 / 1024))\n deleted_file_ids.append(row[\"file_id\"])\n try:\n site.content_manager.optionalRemoved(row[\"inner_path\"], row[\"hash_id\"], row[\"size\"])\n site.storage.delete(row[\"inner_path\"])\n need_delete -= row[\"size\"]\n except Exception as err:\n site.log.error(\"Error deleting %s: %s\" % (row[\"inner_path\"], err))\n\n if need_delete <= 0:\n break\n\n cur = self.getCursor()\n for file_id in deleted_file_ids:\n cur.execute(\"UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?\", {\"file_id\": file_id})\n cur.close()\n\n\n@PluginManager.registerTo(\"SiteManager\")\nclass SiteManagerPlugin(object):\n def load(self, *args, **kwargs):\n back = super(SiteManagerPlugin, self).load(*args, **kwargs)\n if self.sites and not content_db.optional_files_loaded and content_db.conn:\n content_db.optional_files_loaded = True\n content_db.loadFilesOptional()\n return back"
},
{
"path": "plugins/OptionalManager/OptionalManagerPlugin.py",
"content": "import time\nimport re\nimport collections\n\nimport gevent\n\nfrom util import helper\nfrom Plugin import PluginManager\nfrom . import ContentDbPlugin\n\n\n# We can only import plugin host clases after the plugins are loaded\n@PluginManager.afterLoad\ndef importPluginnedClasses():\n global config\n from Config import config\n\n\ndef processAccessLog():\n global access_log\n if access_log:\n content_db = ContentDbPlugin.content_db\n if not content_db.conn:\n return False\n\n s = time.time()\n access_log_prev = access_log\n access_log = collections.defaultdict(dict)\n now = int(time.time())\n num = 0\n for site_id in access_log_prev:\n content_db.execute(\n \"UPDATE file_optional SET time_accessed = %s WHERE ?\" % now,\n {\"site_id\": site_id, \"inner_path\": list(access_log_prev[site_id].keys())}\n )\n num += len(access_log_prev[site_id])\n\n content_db.log.debug(\"Inserted %s web request stat in %.3fs\" % (num, time.time() - s))\n\n\ndef processRequestLog():\n global request_log\n if request_log:\n content_db = ContentDbPlugin.content_db\n if not content_db.conn:\n return False\n\n s = time.time()\n request_log_prev = request_log\n request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}\n num = 0\n for site_id in request_log_prev:\n for inner_path, uploaded in request_log_prev[site_id].items():\n content_db.execute(\n \"UPDATE file_optional SET uploaded = uploaded + %s WHERE ?\" % uploaded,\n {\"site_id\": site_id, \"inner_path\": inner_path}\n )\n num += 1\n content_db.log.debug(\"Inserted %s file request stat in %.3fs\" % (num, time.time() - s))\n\n\nif \"access_log\" not in locals().keys(): # To keep between module reloads\n access_log = collections.defaultdict(dict) # {site_id: {inner_path1: 1, inner_path2: 1...}}\n request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}\n helper.timer(61, processAccessLog)\n helper.timer(60, processRequestLog)\n\n\n@PluginManager.registerTo(\"ContentManager\")\nclass ContentManagerPlugin(object):\n def __init__(self, *args, **kwargs):\n self.cache_is_pinned = {}\n super(ContentManagerPlugin, self).__init__(*args, **kwargs)\n\n def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):\n if \"|\" in inner_path: # Big file piece\n file_inner_path, file_range = inner_path.split(\"|\")\n else:\n file_inner_path = inner_path\n\n self.contents.db.executeDelayed(\n \"UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 0\",\n {\"now\": int(time.time()), \"site_id\": self.contents.db.site_ids[self.site.address], \"inner_path\": file_inner_path}\n )\n\n return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own)\n\n def optionalRemoved(self, inner_path, hash_id, size=None):\n res = self.contents.db.execute(\n \"UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 1\",\n {\"site_id\": self.contents.db.site_ids[self.site.address], \"inner_path\": inner_path}\n )\n\n if res.rowcount > 0:\n back = super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size)\n # Re-add to hashfield if we have other file with the same hash_id\n if self.isDownloaded(hash_id=hash_id, force_check_db=True):\n self.hashfield.appendHashId(hash_id)\n else:\n back = False\n self.cache_is_pinned = {}\n return back\n\n def optionalRenamed(self, inner_path_old, inner_path_new):\n back = super(ContentManagerPlugin, self).optionalRenamed(inner_path_old, inner_path_new)\n self.cache_is_pinned = {}\n self.contents.db.execute(\n \"UPDATE file_optional SET inner_path = :inner_path_new WHERE site_id = :site_id AND inner_path = :inner_path_old\",\n {\"site_id\": self.contents.db.site_ids[self.site.address], \"inner_path_old\": inner_path_old, \"inner_path_new\": inner_path_new}\n )\n return back\n\n def isDownloaded(self, inner_path=None, hash_id=None, force_check_db=False):\n if hash_id and not force_check_db and hash_id not in self.hashfield:\n return False\n\n if inner_path:\n res = self.contents.db.execute(\n \"SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1\",\n {\"site_id\": self.contents.db.site_ids[self.site.address], \"inner_path\": inner_path}\n )\n else:\n res = self.contents.db.execute(\n \"SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND hash_id = :hash_id AND is_downloaded = 1 LIMIT 1\",\n {\"site_id\": self.contents.db.site_ids[self.site.address], \"hash_id\": hash_id}\n )\n row = res.fetchone()\n if row and row[\"is_downloaded\"]:\n return True\n else:\n return False\n\n def isPinned(self, inner_path):\n if inner_path in self.cache_is_pinned:\n self.site.log.debug(\"Cached is pinned: %s\" % inner_path)\n return self.cache_is_pinned[inner_path]\n\n res = self.contents.db.execute(\n \"SELECT is_pinned FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1\",\n {\"site_id\": self.contents.db.site_ids[self.site.address], \"inner_path\": inner_path}\n )\n row = res.fetchone()\n\n if row and row[0]:\n is_pinned = True\n else:\n is_pinned = False\n\n self.cache_is_pinned[inner_path] = is_pinned\n self.site.log.debug(\"Cache set is pinned: %s %s\" % (inner_path, is_pinned))\n\n return is_pinned\n\n def setPin(self, inner_path, is_pinned):\n content_db = self.contents.db\n site_id = content_db.site_ids[self.site.address]\n content_db.execute(\"UPDATE file_optional SET is_pinned = %d WHERE ?\" % is_pinned, {\"site_id\": site_id, \"inner_path\": inner_path})\n self.cache_is_pinned = {}\n\n def optionalDelete(self, inner_path):\n if self.isPinned(inner_path):\n self.site.log.debug(\"Skip deleting pinned optional file: %s\" % inner_path)\n return False\n else:\n return super(ContentManagerPlugin, self).optionalDelete(inner_path)\n\n\n@PluginManager.registerTo(\"WorkerManager\")\nclass WorkerManagerPlugin(object):\n def doneTask(self, task):\n super(WorkerManagerPlugin, self).doneTask(task)\n\n if task[\"optional_hash_id\"] and not self.tasks: # Execute delayed queries immedietly after tasks finished\n ContentDbPlugin.content_db.processDelayed()\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n def parsePath(self, path):\n global access_log\n path_parts = super(UiRequestPlugin, self).parsePath(path)\n if path_parts:\n site_id = ContentDbPlugin.content_db.site_ids.get(path_parts[\"request_address\"])\n if site_id:\n if ContentDbPlugin.content_db.isOptionalFile(site_id, path_parts[\"inner_path\"]):\n access_log[site_id][path_parts[\"inner_path\"]] = 1\n return path_parts\n\n\n@PluginManager.registerTo(\"FileRequest\")\nclass FileRequestPlugin(object):\n def actionGetFile(self, params):\n stats = super(FileRequestPlugin, self).actionGetFile(params)\n self.recordFileRequest(params[\"site\"], params[\"inner_path\"], stats)\n return stats\n\n def actionStreamFile(self, params):\n stats = super(FileRequestPlugin, self).actionStreamFile(params)\n self.recordFileRequest(params[\"site\"], params[\"inner_path\"], stats)\n return stats\n\n def recordFileRequest(self, site_address, inner_path, stats):\n if not stats:\n # Only track the last request of files\n return False\n site_id = ContentDbPlugin.content_db.site_ids[site_address]\n if site_id and ContentDbPlugin.content_db.isOptionalFile(site_id, inner_path):\n request_log[site_id][inner_path] += stats[\"bytes_sent\"]\n\n\n@PluginManager.registerTo(\"Site\")\nclass SitePlugin(object):\n def isDownloadable(self, inner_path):\n is_downloadable = super(SitePlugin, self).isDownloadable(inner_path)\n if is_downloadable:\n return is_downloadable\n\n for path in self.settings.get(\"optional_help\", {}).keys():\n if inner_path.startswith(path):\n return True\n\n return False\n\n def fileForgot(self, inner_path):\n if \"|\" in inner_path and self.content_manager.isPinned(re.sub(r\"\\|.*\", \"\", inner_path)):\n self.log.debug(\"File %s is pinned, no fileForgot\" % inner_path)\n return False\n else:\n return super(SitePlugin, self).fileForgot(inner_path)\n\n def fileDone(self, inner_path):\n if \"|\" in inner_path and self.bad_files.get(inner_path, 0) > 5: # Idle optional file done\n inner_path_file = re.sub(r\"\\|.*\", \"\", inner_path)\n num_changed = 0\n for key, val in self.bad_files.items():\n if key.startswith(inner_path_file) and val > 1:\n self.bad_files[key] = 1\n num_changed += 1\n self.log.debug(\"Idle optional file piece done, changed retry number of %s pieces.\" % num_changed)\n if num_changed:\n gevent.spawn(self.retryBadFiles)\n\n return super(SitePlugin, self).fileDone(inner_path)\n\n\n@PluginManager.registerTo(\"ConfigPlugin\")\nclass ConfigPlugin(object):\n def createArguments(self):\n group = self.parser.add_argument_group(\"OptionalManager plugin\")\n group.add_argument('--optional_limit', help='Limit total size of optional files', default=\"10%\", metavar=\"GB or free space %\")\n group.add_argument('--optional_limit_exclude_minsize', help='Exclude files larger than this limit from optional size limit calculation', default=20, metavar=\"MB\", type=int)\n\n return super(ConfigPlugin, self).createArguments()\n"
},
{
"path": "plugins/OptionalManager/Test/TestOptionalManager.py",
"content": "import copy\n\nimport pytest\n\n\n@pytest.mark.usefixtures(\"resetSettings\")\nclass TestOptionalManager:\n def testDbFill(self, site):\n contents = site.content_manager.contents\n assert len(site.content_manager.hashfield) > 0\n assert contents.db.execute(\"SELECT COUNT(*) FROM file_optional WHERE is_downloaded = 1\").fetchone()[0] == len(site.content_manager.hashfield)\n\n def testSetContent(self, site):\n contents = site.content_manager.contents\n\n # Add new file\n new_content = copy.deepcopy(contents[\"content.json\"])\n new_content[\"files_optional\"][\"testfile\"] = {\n \"size\": 1234,\n \"sha512\": \"aaaabbbbcccc\"\n }\n num_optional_files_before = contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0]\n contents[\"content.json\"] = new_content\n assert contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0] > num_optional_files_before\n\n # Remove file\n new_content = copy.deepcopy(contents[\"content.json\"])\n del new_content[\"files_optional\"][\"testfile\"]\n num_optional_files_before = contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0]\n contents[\"content.json\"] = new_content\n assert contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0] < num_optional_files_before\n\n def testDeleteContent(self, site):\n contents = site.content_manager.contents\n num_optional_files_before = contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0]\n del contents[\"content.json\"]\n assert contents.db.execute(\"SELECT COUNT(*) FROM file_optional\").fetchone()[0] < num_optional_files_before\n\n def testVerifyFiles(self, site):\n contents = site.content_manager.contents\n\n # Add new file\n new_content = copy.deepcopy(contents[\"content.json\"])\n new_content[\"files_optional\"][\"testfile\"] = {\n \"size\": 1234,\n \"sha512\": \"aaaabbbbcccc\"\n }\n contents[\"content.json\"] = new_content\n file_row = contents.db.execute(\"SELECT * FROM file_optional WHERE inner_path = 'testfile'\").fetchone()\n assert not file_row[\"is_downloaded\"]\n\n # Write file from outside of ZeroNet\n site.storage.open(\"testfile\", \"wb\").write(b\"A\" * 1234) # For quick check hash does not matter only file size\n\n hashfield_len_before = len(site.content_manager.hashfield)\n site.storage.verifyFiles(quick_check=True)\n assert len(site.content_manager.hashfield) == hashfield_len_before + 1\n\n file_row = contents.db.execute(\"SELECT * FROM file_optional WHERE inner_path = 'testfile'\").fetchone()\n assert file_row[\"is_downloaded\"]\n\n # Delete file outside of ZeroNet\n site.storage.delete(\"testfile\")\n site.storage.verifyFiles(quick_check=True)\n file_row = contents.db.execute(\"SELECT * FROM file_optional WHERE inner_path = 'testfile'\").fetchone()\n assert not file_row[\"is_downloaded\"]\n\n def testVerifyFilesSameHashId(self, site):\n contents = site.content_manager.contents\n\n new_content = copy.deepcopy(contents[\"content.json\"])\n\n # Add two files with same hashid (first 4 character)\n new_content[\"files_optional\"][\"testfile1\"] = {\n \"size\": 1234,\n \"sha512\": \"aaaabbbbcccc\"\n }\n new_content[\"files_optional\"][\"testfile2\"] = {\n \"size\": 2345,\n \"sha512\": \"aaaabbbbdddd\"\n }\n contents[\"content.json\"] = new_content\n\n assert site.content_manager.hashfield.getHashId(\"aaaabbbbcccc\") == site.content_manager.hashfield.getHashId(\"aaaabbbbdddd\")\n\n # Write files from outside of ZeroNet (For quick check hash does not matter only file size)\n site.storage.open(\"testfile1\", \"wb\").write(b\"A\" * 1234)\n site.storage.open(\"testfile2\", \"wb\").write(b\"B\" * 2345)\n\n site.storage.verifyFiles(quick_check=True)\n\n # Make sure that both is downloaded\n assert site.content_manager.isDownloaded(\"testfile1\")\n assert site.content_manager.isDownloaded(\"testfile2\")\n assert site.content_manager.hashfield.getHashId(\"aaaabbbbcccc\") in site.content_manager.hashfield\n\n # Delete one of the files\n site.storage.delete(\"testfile1\")\n site.storage.verifyFiles(quick_check=True)\n assert not site.content_manager.isDownloaded(\"testfile1\")\n assert site.content_manager.isDownloaded(\"testfile2\")\n assert site.content_manager.hashfield.getHashId(\"aaaabbbbdddd\") in site.content_manager.hashfield\n\n def testIsPinned(self, site):\n assert not site.content_manager.isPinned(\"data/img/zerotalk-upvote.png\")\n site.content_manager.setPin(\"data/img/zerotalk-upvote.png\", True)\n assert site.content_manager.isPinned(\"data/img/zerotalk-upvote.png\")\n\n assert len(site.content_manager.cache_is_pinned) == 1\n site.content_manager.cache_is_pinned = {}\n assert site.content_manager.isPinned(\"data/img/zerotalk-upvote.png\")\n\n def testBigfilePieceReset(self, site):\n site.bad_files = {\n \"data/fake_bigfile.mp4|0-1024\": 10,\n \"data/fake_bigfile.mp4|1024-2048\": 10,\n \"data/fake_bigfile.mp4|2048-3064\": 10\n }\n site.onFileDone(\"data/fake_bigfile.mp4|0-1024\")\n assert site.bad_files[\"data/fake_bigfile.mp4|1024-2048\"] == 1\n assert site.bad_files[\"data/fake_bigfile.mp4|2048-3064\"] == 1\n\n def testOptionalDelete(self, site):\n contents = site.content_manager.contents\n\n site.content_manager.setPin(\"data/img/zerotalk-upvote.png\", True)\n site.content_manager.setPin(\"data/img/zeroid.png\", False)\n new_content = copy.deepcopy(contents[\"content.json\"])\n del new_content[\"files_optional\"][\"data/img/zerotalk-upvote.png\"]\n del new_content[\"files_optional\"][\"data/img/zeroid.png\"]\n\n assert site.storage.isFile(\"data/img/zerotalk-upvote.png\")\n assert site.storage.isFile(\"data/img/zeroid.png\")\n\n site.storage.writeJson(\"content.json\", new_content)\n site.content_manager.loadContent(\"content.json\", force=True)\n\n assert not site.storage.isFile(\"data/img/zeroid.png\")\n assert site.storage.isFile(\"data/img/zerotalk-upvote.png\")\n\n def testOptionalRename(self, site):\n contents = site.content_manager.contents\n\n site.content_manager.setPin(\"data/img/zerotalk-upvote.png\", True)\n new_content = copy.deepcopy(contents[\"content.json\"])\n new_content[\"files_optional\"][\"data/img/zerotalk-upvote-new.png\"] = new_content[\"files_optional\"][\"data/img/zerotalk-upvote.png\"]\n del new_content[\"files_optional\"][\"data/img/zerotalk-upvote.png\"]\n\n assert site.storage.isFile(\"data/img/zerotalk-upvote.png\")\n assert site.content_manager.isPinned(\"data/img/zerotalk-upvote.png\")\n\n site.storage.writeJson(\"content.json\", new_content)\n site.content_manager.loadContent(\"content.json\", force=True)\n\n assert not site.storage.isFile(\"data/img/zerotalk-upvote.png\")\n assert not site.content_manager.isPinned(\"data/img/zerotalk-upvote.png\")\n assert site.content_manager.isPinned(\"data/img/zerotalk-upvote-new.png\")\n assert site.storage.isFile(\"data/img/zerotalk-upvote-new.png\")\n"
},
{
"path": "plugins/OptionalManager/Test/conftest.py",
"content": "from src.Test.conftest import *"
},
{
"path": "plugins/OptionalManager/Test/pytest.ini",
"content": "[pytest]\npython_files = Test*.py\naddopts = -rsxX -v --durations=6\nmarkers =\n webtest: mark a test as a webtest."
},
{
"path": "plugins/OptionalManager/UiWebsocketPlugin.py",
"content": "import re\nimport time\nimport html\nimport os\n\nimport gevent\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom util import helper\nfrom util.Flag import flag\nfrom Translate import Translate\n\n\nplugin_dir = os.path.dirname(__file__)\n\nif \"_\" not in locals():\n _ = Translate(plugin_dir + \"/languages/\")\n\nbigfile_sha512_cache = {}\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def __init__(self, *args, **kwargs):\n self.time_peer_numbers_updated = 0\n super(UiWebsocketPlugin, self).__init__(*args, **kwargs)\n\n def actionSiteSign(self, to, privatekey=None, inner_path=\"content.json\", *args, **kwargs):\n # Add file to content.db and set it as pinned\n content_db = self.site.content_manager.contents.db\n content_inner_dir = helper.getDirname(inner_path)\n content_db.my_optional_files[self.site.address + \"/\" + content_inner_dir] = time.time()\n if len(content_db.my_optional_files) > 50: # Keep only last 50\n oldest_key = min(\n iter(content_db.my_optional_files.keys()),\n key=(lambda key: content_db.my_optional_files[key])\n )\n del content_db.my_optional_files[oldest_key]\n\n return super(UiWebsocketPlugin, self).actionSiteSign(to, privatekey, inner_path, *args, **kwargs)\n\n def updatePeerNumbers(self):\n self.site.updateHashfield()\n content_db = self.site.content_manager.contents.db\n content_db.updatePeerNumbers()\n self.site.updateWebsocket(peernumber_updated=True)\n\n def addBigfileInfo(self, row):\n global bigfile_sha512_cache\n\n content_db = self.site.content_manager.contents.db\n site = content_db.sites[row[\"address\"]]\n if not site.settings.get(\"has_bigfile\"):\n return False\n\n file_key = row[\"address\"] + \"/\" + row[\"inner_path\"]\n sha512 = bigfile_sha512_cache.get(file_key)\n file_info = None\n if not sha512:\n file_info = site.content_manager.getFileInfo(row[\"inner_path\"])\n if not file_info or not file_info.get(\"piece_size\"):\n return False\n sha512 = file_info[\"sha512\"]\n bigfile_sha512_cache[file_key] = sha512\n\n if sha512 in site.storage.piecefields:\n piecefield = site.storage.piecefields[sha512].tobytes()\n else:\n piecefield = None\n\n if piecefield:\n row[\"pieces\"] = len(piecefield)\n row[\"pieces_downloaded\"] = piecefield.count(b\"\\x01\")\n row[\"downloaded_percent\"] = 100 * row[\"pieces_downloaded\"] / row[\"pieces\"]\n if row[\"pieces_downloaded\"]:\n if row[\"pieces\"] == row[\"pieces_downloaded\"]:\n row[\"bytes_downloaded\"] = row[\"size\"]\n else:\n if not file_info:\n file_info = site.content_manager.getFileInfo(row[\"inner_path\"])\n row[\"bytes_downloaded\"] = row[\"pieces_downloaded\"] * file_info.get(\"piece_size\", 0)\n else:\n row[\"bytes_downloaded\"] = 0\n\n row[\"is_downloading\"] = bool(next((inner_path for inner_path in site.bad_files if inner_path.startswith(row[\"inner_path\"])), False))\n\n # Add leech / seed stats\n row[\"peer_seed\"] = 0\n row[\"peer_leech\"] = 0\n for peer in site.peers.values():\n if not peer.time_piecefields_updated or sha512 not in peer.piecefields:\n continue\n peer_piecefield = peer.piecefields[sha512].tobytes()\n if not peer_piecefield:\n continue\n if peer_piecefield == b\"\\x01\" * len(peer_piecefield):\n row[\"peer_seed\"] += 1\n else:\n row[\"peer_leech\"] += 1\n\n # Add myself\n if piecefield:\n if row[\"pieces_downloaded\"] == row[\"pieces\"]:\n row[\"peer_seed\"] += 1\n else:\n row[\"peer_leech\"] += 1\n\n return True\n\n # Optional file functions\n\n def actionOptionalFileList(self, to, address=None, orderby=\"time_downloaded DESC\", limit=10, filter=\"downloaded\", filter_inner_path=None):\n if not address:\n address = self.site.address\n\n # Update peer numbers if necessary\n content_db = self.site.content_manager.contents.db\n if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:\n # Start in new thread to avoid blocking\n self.time_peer_numbers_updated = time.time()\n gevent.spawn(self.updatePeerNumbers)\n\n if address == \"all\" and \"ADMIN\" not in self.permissions:\n return self.response(to, {\"error\": \"Forbidden\"})\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n if not all([re.match(\"^[a-z_*/+-]+( DESC| ASC|)$\", part.strip()) for part in orderby.split(\",\")]):\n return self.response(to, \"Invalid order_by\")\n\n if type(limit) != int:\n return self.response(to, \"Invalid limit\")\n\n back = []\n content_db = self.site.content_manager.contents.db\n\n wheres = {}\n wheres_raw = []\n if \"bigfile\" in filter:\n wheres[\"size >\"] = 1024 * 1024 * 1\n if \"downloaded\" in filter:\n wheres_raw.append(\"(is_downloaded = 1 OR is_pinned = 1)\")\n if \"pinned\" in filter:\n wheres[\"is_pinned\"] = 1\n if filter_inner_path:\n wheres[\"inner_path__like\"] = filter_inner_path\n\n if address == \"all\":\n join = \"LEFT JOIN site USING (site_id)\"\n else:\n wheres[\"site_id\"] = content_db.site_ids[address]\n join = \"\"\n\n if wheres_raw:\n query_wheres_raw = \"AND\" + \" AND \".join(wheres_raw)\n else:\n query_wheres_raw = \"\"\n\n query = \"SELECT * FROM file_optional %s WHERE ? %s ORDER BY %s LIMIT %s\" % (join, query_wheres_raw, orderby, limit)\n\n for row in content_db.execute(query, wheres):\n row = dict(row)\n if address != \"all\":\n row[\"address\"] = address\n\n if row[\"size\"] > 1024 * 1024:\n has_bigfile_info = self.addBigfileInfo(row)\n else:\n has_bigfile_info = False\n\n if not has_bigfile_info and \"bigfile\" in filter:\n continue\n\n if not has_bigfile_info:\n if row[\"is_downloaded\"]:\n row[\"bytes_downloaded\"] = row[\"size\"]\n row[\"downloaded_percent\"] = 100\n else:\n row[\"bytes_downloaded\"] = 0\n row[\"downloaded_percent\"] = 0\n\n back.append(row)\n self.response(to, back)\n\n def actionOptionalFileInfo(self, to, inner_path):\n content_db = self.site.content_manager.contents.db\n site_id = content_db.site_ids[self.site.address]\n\n # Update peer numbers if necessary\n if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:\n # Start in new thread to avoid blocking\n self.time_peer_numbers_updated = time.time()\n gevent.spawn(self.updatePeerNumbers)\n\n query = \"SELECT * FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1\"\n res = content_db.execute(query, {\"site_id\": site_id, \"inner_path\": inner_path})\n row = next(res, None)\n if row:\n row = dict(row)\n if row[\"size\"] > 1024 * 1024:\n row[\"address\"] = self.site.address\n self.addBigfileInfo(row)\n self.response(to, row)\n else:\n self.response(to, None)\n\n def setPin(self, inner_path, is_pinned, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return {\"error\": \"Forbidden\"}\n\n site = self.server.sites[address]\n site.content_manager.setPin(inner_path, is_pinned)\n\n return \"ok\"\n\n @flag.no_multiuser\n def actionOptionalFilePin(self, to, inner_path, address=None):\n if type(inner_path) is not list:\n inner_path = [inner_path]\n back = self.setPin(inner_path, 1, address)\n num_file = len(inner_path)\n if back == \"ok\":\n if num_file == 1:\n self.cmd(\"notification\", [\"done\", _[\"Pinned %s\"] % html.escape(helper.getFilename(inner_path[0])), 5000])\n else:\n self.cmd(\"notification\", [\"done\", _[\"Pinned %s files\"] % num_file, 5000])\n self.response(to, back)\n\n @flag.no_multiuser\n def actionOptionalFileUnpin(self, to, inner_path, address=None):\n if type(inner_path) is not list:\n inner_path = [inner_path]\n back = self.setPin(inner_path, 0, address)\n num_file = len(inner_path)\n if back == \"ok\":\n if num_file == 1:\n self.cmd(\"notification\", [\"done\", _[\"Removed pin from %s\"] % html.escape(helper.getFilename(inner_path[0])), 5000])\n else:\n self.cmd(\"notification\", [\"done\", _[\"Removed pin from %s files\"] % num_file, 5000])\n self.response(to, back)\n\n @flag.no_multiuser\n def actionOptionalFileDelete(self, to, inner_path, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n site = self.server.sites[address]\n\n content_db = site.content_manager.contents.db\n site_id = content_db.site_ids[site.address]\n\n res = content_db.execute(\"SELECT * FROM file_optional WHERE ? LIMIT 1\", {\"site_id\": site_id, \"inner_path\": inner_path, \"is_downloaded\": 1})\n row = next(res, None)\n\n if not row:\n return self.response(to, {\"error\": \"Not found in content.db\"})\n\n removed = site.content_manager.optionalRemoved(inner_path, row[\"hash_id\"], row[\"size\"])\n # if not removed:\n # return self.response(to, {\"error\": \"Not found in hash_id: %s\" % row[\"hash_id\"]})\n\n content_db.execute(\"UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?\", {\"site_id\": site_id, \"inner_path\": inner_path})\n\n try:\n site.storage.delete(inner_path)\n except Exception as err:\n return self.response(to, {\"error\": \"File delete error: %s\" % err})\n site.updateWebsocket(file_delete=inner_path)\n\n if inner_path in site.content_manager.cache_is_pinned:\n site.content_manager.cache_is_pinned = {}\n\n self.response(to, \"ok\")\n\n # Limit functions\n\n @flag.admin\n def actionOptionalLimitStats(self, to):\n back = {}\n back[\"limit\"] = config.optional_limit\n back[\"used\"] = self.site.content_manager.contents.db.getOptionalUsedBytes()\n back[\"free\"] = helper.getFreeSpace()\n\n self.response(to, back)\n\n @flag.no_multiuser\n @flag.admin\n def actionOptionalLimitSet(self, to, limit):\n config.optional_limit = re.sub(r\"\\.0+$\", \"\", limit) # Remove unnecessary digits from end\n config.saveValue(\"optional_limit\", limit)\n self.response(to, \"ok\")\n\n # Distribute help functions\n\n def actionOptionalHelpList(self, to, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n site = self.server.sites[address]\n\n self.response(to, site.settings.get(\"optional_help\", {}))\n\n @flag.no_multiuser\n def actionOptionalHelp(self, to, directory, title, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n site = self.server.sites[address]\n content_db = site.content_manager.contents.db\n site_id = content_db.site_ids[address]\n\n if \"optional_help\" not in site.settings:\n site.settings[\"optional_help\"] = {}\n\n stats = content_db.execute(\n \"SELECT COUNT(*) AS num, SUM(size) AS size FROM file_optional WHERE site_id = :site_id AND inner_path LIKE :inner_path\",\n {\"site_id\": site_id, \"inner_path\": directory + \"%\"}\n ).fetchone()\n stats = dict(stats)\n\n if not stats[\"size\"]:\n stats[\"size\"] = 0\n if not stats[\"num\"]:\n stats[\"num\"] = 0\n\n self.cmd(\"notification\", [\n \"done\",\n _[\"You started to help distribute %s.Directory: %s\"] %\n (html.escape(title), html.escape(directory)),\n 10000\n ])\n\n site.settings[\"optional_help\"][directory] = title\n\n self.response(to, dict(stats))\n\n @flag.no_multiuser\n def actionOptionalHelpRemove(self, to, directory, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n site = self.server.sites[address]\n\n try:\n del site.settings[\"optional_help\"][directory]\n self.response(to, \"ok\")\n except Exception:\n self.response(to, {\"error\": \"Not found\"})\n\n def cbOptionalHelpAll(self, to, site, value):\n site.settings[\"autodownloadoptional\"] = value\n self.response(to, value)\n\n @flag.no_multiuser\n def actionOptionalHelpAll(self, to, value, address=None):\n if not address:\n address = self.site.address\n\n if not self.hasSitePermission(address):\n return self.response(to, {\"error\": \"Forbidden\"})\n\n site = self.server.sites[address]\n\n if value:\n if \"ADMIN\" in self.site.settings[\"permissions\"]:\n self.cbOptionalHelpAll(to, site, True)\n else:\n site_title = site.content_manager.contents[\"content.json\"].get(\"title\", address)\n self.cmd(\n \"confirm\",\n [\n _[\"Help distribute all new optional files on site %s\"] % html.escape(site_title),\n _[\"Yes, I want to help!\"]\n ],\n lambda res: self.cbOptionalHelpAll(to, site, True)\n )\n else:\n site.settings[\"autodownloadoptional\"] = False\n self.response(to, False)\n" }, { "path": "plugins/OptionalManager/__init__.py", "content": "from . import OptionalManagerPlugin\nfrom . import UiWebsocketPlugin\n" }, { "path": "plugins/OptionalManager/languages/es.json", "content": "{\n\t\"Pinned %s files\": \"Archivos %s fijados\",\n\t\"Removed pin from %s files\": \"Archivos %s que no estan fijados\",\n\t\"You started to help distribute %s.
Directory: %s\": \"Tu empezaste a ayudar a distribuir %s.
Directorio: %s\",\n\t\"Help distribute all new optional files on site %s\": \"Ayude a distribuir todos los archivos opcionales en el sitio %s\",\n\t\"Yes, I want to help!\": \"¡Si, yo quiero ayudar!\"\n}\n" }, { "path": "plugins/OptionalManager/languages/fr.json", "content": "{\n\t\"Pinned %s files\": \"Fichiers %s épinglés\",\n\t\"Removed pin from %s files\": \"Fichiers %s ne sont plus épinglés\",\n\t\"You started to help distribute %s.
Directory: %s\": \"Vous avez commencé à aider à distribuer %s.
Dossier : %s\",\n\t\"Help distribute all new optional files on site %s\": \"Aider à distribuer tous les fichiers optionnels du site %s\",\n\t\"Yes, I want to help!\": \"Oui, je veux aider !\"\n}\n" }, { "path": "plugins/OptionalManager/languages/hu.json", "content": "{\n\t\"Pinned %s files\": \"%s fájl rögzítve\",\n\t\"Removed pin from %s files\": \"%s fájl rögzítés eltávolítva\",\n\t\"You started to help distribute %s.
Directory: %s\": \"Új segítség a terjesztésben: %s.
Könyvtár: %s\",\n\t\"Help distribute all new optional files on site %s\": \"Segítség az összes új opcionális fájl terjesztésében az %s oldalon\",\n\t\"Yes, I want to help!\": \"Igen, segíteni akarok!\"\n}\n" }, { "path": "plugins/OptionalManager/languages/jp.json", "content": "{\n\t\"Pinned %s files\": \"%s 件のファイルを固定\",\n\t\"Removed pin from %s files\": \"%s 件のファイルの固定を解除\",\n\t\"You started to help distribute %s.
Directory: %s\": \"あなたはサイト: %s の配布の援助を開始しました。
ディレクトリ: %s\",\n\t\"Help distribute all new optional files on site %s\": \"サイト: %s のすべての新しいオプションファイルの配布を援助しますか？\",\n\t\"Yes, I want to help!\": \"はい、やります！\"\n}\n" }, { "path": "plugins/OptionalManager/languages/pt-br.json", "content": "{\n\t\"Pinned %s files\": \"Arquivos %s fixados\",\n\t\"Removed pin from %s files\": \"Arquivos %s não estão fixados\",\n\t\"You started to help distribute %s.
Directory: %s\": \"Você começou a ajudar a distribuir %s.
Pasta: %s\",\n\t\"Help distribute all new optional files on site %s\": \"Ajude a distribuir todos os novos arquivos opcionais no site %s\",\n\t\"Yes, I want to help!\": \"Sim, eu quero ajudar!\"\n}\n" }, { "path": "plugins/OptionalManager/languages/zh-tw.json", "content": "{\n\t\"Pinned %s files\": \"已固定 %s 個檔\",\n\t\"Removed pin from %s files\": \"已解除固定 %s 個檔\",\n\t\"You started to help distribute %s.
Directory: %s\": \"你已經開始幫助分發 %s 。
目錄：%s\",\n\t\"Help distribute all new optional files on site %s\": \"你想要幫助分發 %s 網站的所有檔嗎？\",\n\t\"Yes, I want to help!\": \"是，我想要幫助！\"\n}\n" }, { "path": "plugins/OptionalManager/languages/zh.json", "content": "{\n\t\"Pinned %s files\": \"已固定 %s 个文件\",\n\t\"Removed pin from %s files\": \"已解除固定 %s 个文件\",\n\t\"You started to help distribute %s.
Directory: %s\": \"您已经开始帮助分发 %s 。
目录：%s\",\n\t\"Help distribute all new optional files on site %s\": \"您想要帮助分发 %s 站点的所有文件吗？\",\n\t\"Yes, I want to help!\": \"是，我想要帮助！\"\n}\n" }, { "path": "plugins/PeerDb/PeerDbPlugin.py", "content": "import time\nimport sqlite3\nimport random\nimport atexit\n\nimport gevent\nfrom Plugin import PluginManager\n\n\n@PluginManager.registerTo(\"ContentDb\")\nclass ContentDbPlugin(object):\n def __init__(self, *args, **kwargs):\n atexit.register(self.saveAllPeers)\n super(ContentDbPlugin, self).__init__(*args, **kwargs)\n\n def getSchema(self):\n schema = super(ContentDbPlugin, self).getSchema()\n\n schema[\"tables\"][\"peer\"] = {\n \"cols\": [\n [\"site_id\", \"INTEGER REFERENCES site (site_id) ON DELETE CASCADE\"],\n [\"address\", \"TEXT NOT NULL\"],\n [\"port\", \"INTEGER NOT NULL\"],\n [\"hashfield\", \"BLOB\"],\n [\"reputation\", \"INTEGER NOT NULL\"],\n [\"time_added\", \"INTEGER NOT NULL\"],\n [\"time_found\", \"INTEGER NOT NULL\"]\n ],\n \"indexes\": [\n \"CREATE UNIQUE INDEX peer_key ON peer (site_id, address, port)\"\n ],\n \"schema_changed\": 2\n }\n\n return schema\n\n def loadPeers(self, site):\n s = time.time()\n site_id = self.site_ids.get(site.address)\n res = self.execute(\"SELECT * FROM peer WHERE site_id = :site_id\", {\"site_id\": site_id})\n num = 0\n num_hashfield = 0\n for row in res:\n peer = site.addPeer(str(row[\"address\"]), row[\"port\"])\n if not peer: # Already exist\n continue\n if row[\"hashfield\"]:\n peer.hashfield.replaceFromBytes(row[\"hashfield\"])\n num_hashfield += 1\n peer.time_added = row[\"time_added\"]\n peer.time_found = row[\"time_found\"]\n peer.reputation = row[\"reputation\"]\n if row[\"address\"].endswith(\".onion\"):\n peer.reputation = peer.reputation / 2 - 1 # Onion peers less likely working\n num += 1\n if num_hashfield:\n site.content_manager.has_optional_files = True\n site.log.debug(\"%s peers (%s with hashfield) loaded in %.3fs\" % (num, num_hashfield, time.time() - s))\n\n def iteratePeers(self, site):\n site_id = self.site_ids.get(site.address)\n for key, peer in list(site.peers.items()):\n address, port = key.rsplit(\":\", 1)\n if peer.has_hashfield:\n hashfield = sqlite3.Binary(peer.hashfield.tobytes())\n else:\n hashfield = \"\"\n yield (site_id, address, port, hashfield, peer.reputation, int(peer.time_added), int(peer.time_found))\n\n def savePeers(self, site, spawn=False):\n if spawn:\n # Save peers every hour (+random some secs to not update very site at same time)\n site.greenlet_manager.spawnLater(60 * 60 + random.randint(0, 60), self.savePeers, site, spawn=True)\n if not site.peers:\n site.log.debug(\"Peers not saved: No peers found\")\n return\n s = time.time()\n site_id = self.site_ids.get(site.address)\n cur = self.getCursor()\n try:\n cur.execute(\"DELETE FROM peer WHERE site_id = :site_id\", {\"site_id\": site_id})\n cur.executemany(\n \"INSERT INTO peer (site_id, address, port, hashfield, reputation, time_added, time_found) VALUES (?, ?, ?, ?, ?, ?, ?)\",\n self.iteratePeers(site)\n )\n except Exception as err:\n site.log.error(\"Save peer error: %s\" % err)\n site.log.debug(\"Peers saved in %.3fs\" % (time.time() - s))\n\n def initSite(self, site):\n super(ContentDbPlugin, self).initSite(site)\n site.greenlet_manager.spawnLater(0.5, self.loadPeers, site)\n site.greenlet_manager.spawnLater(60*60, self.savePeers, site, spawn=True)\n\n def saveAllPeers(self):\n for site in list(self.sites.values()):\n try:\n self.savePeers(site)\n except Exception as err:\n site.log.error(\"Save peer error: %s\" % err)\n" }, { "path": "plugins/PeerDb/__init__.py", "content": "from . import PeerDbPlugin\n\n" }, { "path": "plugins/PeerDb/plugin_info.json", "content": "{\n\t\"name\": \"PeerDb\",\n\t\"description\": \"Save/restore peer list on client restart.\",\n\t\"default\": \"enabled\"\n}" }, { "path": "plugins/Sidebar/ConsolePlugin.py", "content": "import re\nimport logging\n\nfrom Plugin import PluginManager\nfrom Config import config\nfrom Debug import Debug\nfrom util import SafeRe\nfrom util.Flag import flag\n\n\nclass WsLogStreamer(logging.StreamHandler):\n def __init__(self, stream_id, ui_websocket, filter):\n self.stream_id = stream_id\n self.ui_websocket = ui_websocket\n\n if filter:\n if not SafeRe.isSafePattern(filter):\n raise Exception(\"Not a safe prex pattern\")\n self.filter_re = re.compile(\".*\" + filter)\n else:\n self.filter_re = None\n return super(WsLogStreamer, self).__init__()\n\n def emit(self, record):\n if self.ui_websocket.ws.closed:\n self.stop()\n return\n line = self.format(record)\n if self.filter_re and not self.filter_re.match(line):\n return False\n\n self.ui_websocket.cmd(\"logLineAdd\", {\"stream_id\": self.stream_id, \"lines\": [line]})\n\n def stop(self):\n logging.getLogger('').removeHandler(self)\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def __init__(self, *args, **kwargs):\n self.log_streamers = {}\n return super(UiWebsocketPlugin, self).__init__(*args, **kwargs)\n\n @flag.no_multiuser\n @flag.admin\n def actionConsoleLogRead(self, to, filter=None, read_size=32 * 1024, limit=500):\n log_file_path = \"%s/debug.log\" % config.log_dir\n log_file = open(log_file_path, encoding=\"utf-8\")\n log_file.seek(0, 2)\n end_pos = log_file.tell()\n log_file.seek(max(0, end_pos - read_size))\n if log_file.tell() != 0:\n log_file.readline() # Partial line junk\n\n pos_start = log_file.tell()\n lines = []\n if filter:\n assert SafeRe.isSafePattern(filter)\n filter_re = re.compile(\".*\" + filter)\n\n last_match = False\n for line in log_file:\n if not line.startswith(\"[\") and last_match: # Multi-line log entry\n lines.append(line.replace(\" \", \" \"))\n continue\n\n if filter and not filter_re.match(line):\n last_match = False\n continue\n last_match = True\n lines.append(line)\n\n num_found = len(lines)\n lines = lines[-limit:]\n\n return {\"lines\": lines, \"pos_end\": log_file.tell(), \"pos_start\": pos_start, \"num_found\": num_found}\n\n def addLogStreamer(self, stream_id, filter=None):\n logger = WsLogStreamer(stream_id, self, filter)\n logger.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)-8s %(name)s %(message)s'))\n logger.setLevel(logging.getLevelName(\"DEBUG\"))\n\n logging.getLogger('').addHandler(logger)\n return logger\n\n @flag.no_multiuser\n @flag.admin\n def actionConsoleLogStream(self, to, filter=None):\n stream_id = to\n self.log_streamers[stream_id] = self.addLogStreamer(stream_id, filter)\n self.response(to, {\"stream_id\": stream_id})\n\n @flag.no_multiuser\n @flag.admin\n def actionConsoleLogStreamRemove(self, to, stream_id):\n try:\n self.log_streamers[stream_id].stop()\n del self.log_streamers[stream_id]\n return \"ok\"\n except Exception as err:\n return {\"error\": Debug.formatException(err)}\n" }, { "path": "plugins/Sidebar/SidebarPlugin.py", "content": "import re\nimport os\nimport html\nimport sys\nimport math\nimport time\nimport json\nimport io\nimport urllib\nimport urllib.parse\n\nimport gevent\n\nimport util\nfrom Config import config\nfrom Plugin import PluginManager\nfrom Debug import Debug\nfrom Translate import Translate\nfrom util import helper\nfrom util.Flag import flag\nfrom .ZipStream import ZipStream\n\nplugin_dir = os.path.dirname(__file__)\nmedia_dir = plugin_dir + \"/media\"\n\nloc_cache = {}\nif \"_\" not in locals():\n _ = Translate(plugin_dir + \"/languages/\")\n\n\n@PluginManager.registerTo(\"UiRequest\")\nclass UiRequestPlugin(object):\n # Inject our resources to end of original file streams\n def actionUiMedia(self, path):\n if path == \"/uimedia/all.js\" or path == \"/uimedia/all.css\":\n # First yield the original file and header\n body_generator = super(UiRequestPlugin, self).actionUiMedia(path)\n for part in body_generator:\n yield part\n\n # Append our media file to the end\n ext = re.match(\".*(js|css)$\", path).group(1)\n plugin_media_file = \"%s/all.%s\" % (media_dir, ext)\n if config.debug:\n # If debugging merge *.css to all.css and *.js to all.js\n from Debug import DebugMedia\n DebugMedia.merge(plugin_media_file)\n if ext == \"js\":\n yield _.translateData(open(plugin_media_file).read()).encode(\"utf8\")\n else:\n for part in self.actionFile(plugin_media_file, send_header=False):\n yield part\n elif path.startswith(\"/uimedia/globe/\"): # Serve WebGL globe files\n file_name = re.match(\".*/(.*)\", path).group(1)\n plugin_media_file = \"%s_globe/%s\" % (media_dir, file_name)\n if config.debug and path.endswith(\"all.js\"):\n # If debugging merge *.css to all.css and *.js to all.js\n from Debug import DebugMedia\n DebugMedia.merge(plugin_media_file)\n for part in self.actionFile(plugin_media_file):\n yield part\n else:\n for part in super(UiRequestPlugin, self).actionUiMedia(path):\n yield part\n\n def actionZip(self):\n address = self.get[\"address\"]\n site = self.server.site_manager.get(address)\n if not site:\n return self.error404(\"Site not found\")\n\n title = site.content_manager.contents.get(\"content.json\", {}).get(\"title\", \"\")\n filename = \"%s-backup-%s.zip\" % (title, time.strftime(\"%Y-%m-%d_%H_%M\"))\n filename_quoted = urllib.parse.quote(filename)\n self.sendHeader(content_type=\"application/zip\", extra_headers={'Content-Disposition': 'attachment; filename=\"%s\"' % filename_quoted})\n\n return self.streamZip(site.storage.getPath(\".\"))\n\n def streamZip(self, dir_path):\n zs = ZipStream(dir_path)\n while 1:\n data = zs.read()\n if not data:\n break\n yield data\n\n\n@PluginManager.registerTo(\"UiWebsocket\")\nclass UiWebsocketPlugin(object):\n def sidebarRenderPeerStats(self, body, site):\n connected = len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected])\n connectable = len([peer_id for peer_id in list(site.peers.keys()) if not peer_id.endswith(\":0\")])\n onion = len([peer_id for peer_id in list(site.peers.keys()) if \".onion\" in peer_id])\n local = len([peer for peer in list(site.peers.values()) if helper.isPrivateIp(peer.ip)])\n peers_total = len(site.peers)\n\n # Add myself\n if site.isServing():\n peers_total += 1\n if any(site.connection_server.port_opened.values()):\n connectable += 1\n if site.connection_server.tor_manager.start_onions:\n onion += 1\n\n if peers_total:\n percent_connected = float(connected) / peers_total\n percent_connectable = float(connectable) / peers_total\n percent_onion = float(onion) / peers_total\n else:\n percent_connectable = percent_connected = percent_onion = 0\n\n if local:\n local_html = _(\"
- \n \n \n \n \n
- \n
- {_[Connected]}:{connected} \n
- {_[Connectable]}:{connectable} \n
- {_[Onion]}:{onion} \n {local_html}\n
- {_[Total]}:{peers_total} \n
- \n \n \n
- \n
- {_[Received]}:{recv:.2f}MB \n
- {_[Sent]}:{sent:.2f}MB \n
- \n \"\"\"))\n\n extensions = (\n (\"html\", \"yellow\"),\n (\"css\", \"orange\"),\n (\"js\", \"purple\"),\n (\"Image\", \"green\"),\n (\"json\", \"darkblue\"),\n (\"User data\", \"blue\"),\n (\"Other\", \"white\"),\n (\"Total\", \"black\")\n )\n # Collect stats\n size_filetypes = {}\n size_total = 0\n contents = site.content_manager.listContents() # Without user files\n for inner_path in contents:\n content = site.content_manager.contents[inner_path]\n if \"files\" not in content or content[\"files\"] is None:\n continue\n for file_name, file_details in list(content[\"files\"].items()):\n size_total += file_details[\"size\"]\n ext = file_name.split(\".\")[-1]\n size_filetypes[ext] = size_filetypes.get(ext, 0) + file_details[\"size\"]\n\n # Get user file sizes\n size_user_content = site.content_manager.contents.execute(\n \"SELECT SUM(size) + SUM(size_files) AS size FROM content WHERE ?\",\n {\"not__inner_path\": contents}\n ).fetchone()[\"size\"]\n if not size_user_content:\n size_user_content = 0\n size_filetypes[\"User data\"] = size_user_content\n size_total += size_user_content\n\n # The missing difference is content.json sizes\n if \"json\" in size_filetypes:\n size_filetypes[\"json\"] += max(0, site.settings[\"size\"] - size_total)\n size_total = size_other = site.settings[\"size\"]\n\n # Bar\n for extension, color in extensions:\n if extension == \"Total\":\n continue\n if extension == \"Other\":\n size = max(0, size_other)\n elif extension == \"Image\":\n size = size_filetypes.get(\"jpg\", 0) + size_filetypes.get(\"png\", 0) + size_filetypes.get(\"gif\", 0)\n size_other -= size\n else:\n size = size_filetypes.get(extension, 0)\n size_other -= size\n if size_total == 0:\n percent = 0\n else:\n percent = 100 * (float(size) / size_total)\n percent = math.floor(percent * 100) / 100 # Floor to 2 digits\n body.append(\n \"\"\"\"\"\" %\n (percent, _[extension], color, _[extension])\n )\n\n # Legend\n body.append(\"
- \")\n for extension, color in extensions:\n if extension == \"Other\":\n size = max(0, size_other)\n elif extension == \"Image\":\n size = size_filetypes.get(\"jpg\", 0) + size_filetypes.get(\"png\", 0) + size_filetypes.get(\"gif\", 0)\n elif extension == \"Total\":\n size = size_total\n else:\n size = size_filetypes.get(extension, 0)\n\n if extension == \"js\":\n title = \"javascript\"\n else:\n title = extension\n\n if size > 1024 * 1024 * 10: # Format as mB is more than 10mB\n size_formatted = \"%.0fMB\" % (size / 1024 / 1024)\n else:\n size_formatted = \"%.0fkB\" % (size / 1024)\n\n body.append(\"
- %s:%s \" % (color, _[title], size_formatted))\n\n body.append(\"
- \n \n \n
- \n
- {_[Downloaded]}:{size_formatted_downloaded:.2f}MB \n
- {_[Total]}:{size_formatted_total:.2f}MB \n
- \n \"\"\"))\n\n i = 0\n for bad_file, tries in site.bad_files.items():\n i += 1\n body.append(_(\"\"\"
- {bad_filename} \"\"\", {\n \"bad_file_path\": bad_file,\n \"bad_filename\": helper.getFilename(bad_file),\n \"tries\": _.pluralize(tries, \"{} try\", \"{} tries\")\n }))\n if i > 30:\n break\n\n if len(site.bad_files) > 30:\n num_bad_files = len(site.bad_files) - 30\n body.append(_(\"\"\"
- {_[+ {num_bad_files} more]} \"\"\", nested=True))\n\n body.append(\"\"\"\n
\n \n {_[Reload]}\n {_[Rebuild]}\n
\n \n {auth_address}\n {_[Change]}\n
\n \n
\n {site_address}\n \"\"\"))\n if donate_key == False or donate_key == \"\":\n pass\n elif (type(donate_key) == str or type(donate_key) == str) and len(donate_key) > 0:\n body.append(_(\"\"\"\n
\n \n
\n {donate_key}\n \"\"\"))\n else:\n body.append(_(\"\"\"\n {_[Donate]}\n \"\"\"))\n body.append(_(\"\"\"\n
\n {_[This is my site]}
\n \n \"\"\"))\n\n def sidebarRenderOwnSettings(self, body, site):\n title = site.content_manager.contents.get(\"content.json\", {}).get(\"title\", \"\")\n description = site.content_manager.contents.get(\"content.json\", {}).get(\"description\", \"\")\n\n body.append(_(\"\"\"\n\n \n {_[Sign and publish]}\n \\u22EE\n
\n \"\"\"))\n\n contents = [\"content.json\"]\n contents += list(site.content_manager.contents.get(\"content.json\", {}).get(\"includes\", {}).keys())\n body.append(_(\"{_[Choose]}: \"))\n for content in contents:\n body.append(_(\"{content} \"))\n body.append(\"
\")\n body.append(\"\")\n body.append(\"×\")\n body.append(\"
\")\n\n body.append(\"\")\n\n self.response(to, \"\".join(body))\n\n def downloadGeoLiteDb(self, db_path):\n import gzip\n import shutil\n from util import helper\n\n if config.offline:\n return False\n\n self.log.info(\"Downloading GeoLite2 City database...\")\n self.cmd(\"progress\", [\"geolite-info\", _[\"Downloading GeoLite2 City database (one time only, ~20MB)...\"], 0])\n db_urls = [\n \"https://raw.githubusercontent.com/aemr3/GeoLite2-Database/master/GeoLite2-City.mmdb.gz\",\n \"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz\"\n ]\n for db_url in db_urls:\n downloadl_err = None\n try:\n # Download\n response = helper.httpRequest(db_url)\n data_size = response.getheader('content-length')\n data_recv = 0\n data = io.BytesIO()\n while True:\n buff = response.read(1024 * 512)\n if not buff:\n break\n data.write(buff)\n data_recv += 1024 * 512\n if data_size:\n progress = int(float(data_recv) / int(data_size) * 100)\n self.cmd(\"progress\", [\"geolite-info\", _[\"Downloading GeoLite2 City database (one time only, ~20MB)...\"], progress])\n self.log.info(\"GeoLite2 City database downloaded (%s bytes), unpacking...\" % data.tell())\n data.seek(0)\n\n # Unpack\n with gzip.GzipFile(fileobj=data) as gzip_file:\n shutil.copyfileobj(gzip_file, open(db_path, \"wb\"))\n\n self.cmd(\"progress\", [\"geolite-info\", _[\"GeoLite2 City database downloaded!\"], 100])\n time.sleep(2) # Wait for notify animation\n self.log.info(\"GeoLite2 City database is ready at: %s\" % db_path)\n return True\n except Exception as err:\n download_err = err\n self.log.error(\"Error downloading %s: %s\" % (db_url, err))\n pass\n self.cmd(\"progress\", [\n \"geolite-info\",\n _[\"GeoLite2 City database download error: {}!%s
\" % html.escape(site.content_manager.contents.get(\"content.json\", {}).get(\"title\", \"\"), True))\n\n body.append(\"\")\n\n body.append(\"- \")\n\n self.sidebarRenderPeerStats(body, site)\n self.sidebarRenderTransferStats(body, site)\n self.sidebarRenderFileStats(body, site)\n self.sidebarRenderSizeLimit(body, site)\n has_optional = self.sidebarRenderOptionalFileStats(body, site)\n if has_optional:\n self.sidebarRenderOptionalFileSettings(body, site)\n self.sidebarRenderDbOptions(body, site)\n self.sidebarRenderIdentity(body, site)\n self.sidebarRenderControls(body, site)\n if site.bad_files:\n self.sidebarRenderBadFiles(body, site)\n\n self.sidebarRenderOwnedCheckbox(body, site)\n body.append(\"
\")\n self.sidebarRenderOwnSettings(body, site)\n self.sidebarRenderContents(body, site)\n body.append(\"
\")\n body.append(\"Please download manually and unpack to data dir:
{}\"].format(download_err, db_urls[0]),\n -100\n ])\n\n def getLoc(self, geodb, ip):\n global loc_cache\n\n if ip in loc_cache:\n return loc_cache[ip]\n else:\n try:\n loc_data = geodb.get(ip)\n except:\n loc_data = None\n\n if not loc_data or \"location\" not in loc_data:\n loc_cache[ip] = None\n return None\n\n loc = {\n \"lat\": loc_data[\"location\"][\"latitude\"],\n \"lon\": loc_data[\"location\"][\"longitude\"],\n }\n if \"city\" in loc_data:\n loc[\"city\"] = loc_data[\"city\"][\"names\"][\"en\"]\n\n if \"country\" in loc_data:\n loc[\"country\"] = loc_data[\"country\"][\"names\"][\"en\"]\n\n loc_cache[ip] = loc\n return loc\n\n @util.Noparallel()\n def getGeoipDb(self):\n db_name = 'GeoLite2-City.mmdb'\n\n sys_db_paths = []\n if sys.platform == \"linux\":\n sys_db_paths += ['/usr/share/GeoIP/' + db_name]\n\n data_dir_db_path = os.path.join(config.data_dir, db_name)\n\n db_paths = sys_db_paths + [data_dir_db_path]\n\n for path in db_paths:\n if os.path.isfile(path) and os.path.getsize(path) > 0:\n return path\n\n self.log.info(\"GeoIP database not found at [%s]. Downloading to: %s\",\n \" \".join(db_paths), data_dir_db_path)\n if self.downloadGeoLiteDb(data_dir_db_path):\n return data_dir_db_path\n return None\n\n def getPeerLocations(self, peers):\n import maxminddb\n\n db_path = self.getGeoipDb()\n if not db_path:\n self.log.debug(\"Not showing peer locations: no GeoIP database\")\n return False\n\n geodb = maxminddb.open_database(db_path)\n\n peers = list(peers.values())\n # Place bars\n peer_locations = []\n placed = {} # Already placed bars here\n for peer in peers:\n # Height of bar\n if peer.connection and peer.connection.last_ping_delay:\n ping = round(peer.connection.last_ping_delay * 1000)\n else:\n ping = None\n loc = self.getLoc(geodb, peer.ip)\n\n if not loc:\n continue\n # Create position array\n lat, lon = loc[\"lat\"], loc[\"lon\"]\n latlon = \"%s,%s\" % (lat, lon)\n if latlon in placed and helper.getIpType(peer.ip) == \"ipv4\": # Dont place more than 1 bar to same place, fake repos using ip address last two part\n lat += float(128 - int(peer.ip.split(\".\")[-2])) / 50\n lon += float(128 - int(peer.ip.split(\".\")[-1])) / 50\n latlon = \"%s,%s\" % (lat, lon)\n placed[latlon] = True\n peer_location = {}\n peer_location.update(loc)\n peer_location[\"lat\"] = lat\n peer_location[\"lon\"] = lon\n peer_location[\"ping\"] = ping\n\n peer_locations.append(peer_location)\n\n # Append myself\n for ip in self.site.connection_server.ip_external_list:\n my_loc = self.getLoc(geodb, ip)\n if my_loc:\n my_loc[\"ping\"] = 0\n peer_locations.append(my_loc)\n\n return peer_locations\n\n @flag.admin\n @flag.async_run\n def actionSidebarGetPeers(self, to):\n try:\n peer_locations = self.getPeerLocations(self.site.peers)\n globe_data = []\n ping_times = [\n peer_location[\"ping\"]\n for peer_location in peer_locations\n if peer_location[\"ping\"]\n ]\n if ping_times:\n ping_avg = sum(ping_times) / float(len(ping_times))\n else:\n ping_avg = 0\n\n for peer_location in peer_locations:\n if peer_location[\"ping\"] == 0: # Me\n height = -0.135\n elif peer_location[\"ping\"]:\n height = min(0.20, math.log(1 + peer_location[\"ping\"] / ping_avg, 300))\n else:\n height = -0.03\n\n globe_data += [peer_location[\"lat\"], peer_location[\"lon\"], height]\n\n self.response(to, globe_data)\n except Exception as err:\n self.log.debug(\"sidebarGetPeers error: %s\" % Debug.formatException(err))\n self.response(to, {\"error\": str(err)})\n\n @flag.admin\n @flag.no_multiuser\n def actionSiteSetOwned(self, to, owned):\n if self.site.address == config.updatesite:\n return {\"error\": \"You can't change the ownership of the updater site\"}\n\n self.site.settings[\"own\"] = bool(owned)\n self.site.updateWebsocket(owned=owned)\n return \"ok\"\n\n @flag.admin\n @flag.no_multiuser\n def actionSiteRecoverPrivatekey(self, to):\n from Crypt import CryptBitcoin\n\n site_data = self.user.sites[self.site.address]\n if site_data.get(\"privatekey\"):\n return {\"error\": \"This site already has saved privated key\"}\n\n address_index = self.site.content_manager.contents.get(\"content.json\", {}).get(\"address_index\")\n if not address_index:\n return {\"error\": \"No address_index in content.json\"}\n\n privatekey = CryptBitcoin.hdPrivatekey(self.user.master_seed, address_index)\n privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)\n\n if privatekey_address == self.site.address:\n site_data[\"privatekey\"] = privatekey\n self.user.save()\n self.site.updateWebsocket(recover_privatekey=True)\n return \"ok\"\n else:\n return {\"error\": \"Unable to deliver private key for this site from current user's master_seed\"}\n\n @flag.admin\n @flag.no_multiuser\n def actionUserSetSitePrivatekey(self, to, privatekey):\n site_data = self.user.sites[self.site.address]\n site_data[\"privatekey\"] = privatekey\n self.site.updateWebsocket(set_privatekey=bool(privatekey))\n self.user.save()\n\n return \"ok\"\n\n @flag.admin\n @flag.no_multiuser\n def actionSiteSetAutodownloadoptional(self, to, owned):\n self.site.settings[\"autodownloadoptional\"] = bool(owned)\n self.site.worker_manager.removeSolvedFileTasks()\n\n @flag.no_multiuser\n @flag.admin\n def actionDbReload(self, to):\n self.site.storage.closeDb()\n self.site.storage.getDb()\n\n return self.response(to, \"ok\")\n\n @flag.no_multiuser\n @flag.admin\n def actionDbRebuild(self, to):\n try:\n self.site.storage.rebuildDb()\n except Exception as err:\n return self.response(to, {\"error\": str(err)})\n\n\n return self.response(to, \"ok\")\n" }, { "path": "plugins/Sidebar/ZipStream.py", "content": "import io\nimport os\nimport zipfile\n\nclass ZipStream(object):\n def __init__(self, dir_path):\n self.dir_path = dir_path\n self.pos = 0\n self.buff_pos = 0\n self.zf = zipfile.ZipFile(self, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)\n self.buff = io.BytesIO()\n self.file_list = self.getFileList()\n\n def getFileList(self):\n for root, dirs, files in os.walk(self.dir_path):\n for file in files:\n file_path = root + \"/\" + file\n relative_path = os.path.join(os.path.relpath(root, self.dir_path), file)\n yield file_path, relative_path\n self.zf.close()\n\n def read(self, size=60 * 1024):\n for file_path, relative_path in self.file_list:\n self.zf.write(file_path, relative_path)\n if self.buff.tell() >= size:\n break\n self.buff.seek(0)\n back = self.buff.read()\n self.buff.truncate(0)\n self.buff.seek(0)\n self.buff_pos += len(back)\n return back\n\n def write(self, data):\n self.pos += len(data)\n self.buff.write(data)\n\n def tell(self):\n return self.pos\n\n def seek(self, pos, whence=0):\n if pos >= self.buff_pos:\n self.buff.seek(pos - self.buff_pos, whence)\n self.pos = pos\n\n def flush(self):\n pass\n\n\nif __name__ == \"__main__\":\n zs = ZipStream(\".\")\n out = open(\"out.zip\", \"wb\")\n while 1:\n data = zs.read()\n print(\"Write %s\" % len(data))\n if not data:\n break\n out.write(data)\n out.close()\n" }, { "path": "plugins/Sidebar/__init__.py", "content": "from . import SidebarPlugin\nfrom . import ConsolePlugin" }, { "path": "plugins/Sidebar/languages/da.json", "content": "{\n\t\"Peers\": \"Klienter\",\n\t\"Connected\": \"Forbundet\",\n\t\"Connectable\": \"Mulige\",\n\t\"Connectable peers\": \"Mulige klienter\",\n\n\t\"Data transfer\": \"Data overførsel\",\n\t\"Received\": \"Modtaget\",\n\t\"Received bytes\": \"Bytes modtaget\",\n\t\"Sent\": \"Sendt\",\n\t\"Sent bytes\": \"Bytes sendt\",\n\n\t\"Files\": \"Filer\",\n\t\"Total\": \"I alt\",\n\t\"Image\": \"Image\",\n\t\"Other\": \"Andet\",\n\t\"User data\": \"Bruger data\",\n\n\t\"Size limit\": \"Side max størrelse\",\n\t\"limit used\": \"brugt\",\n\t\"free space\": \"fri\",\n\t\"Set\": \"Opdater\",\n\n\t\"Optional files\": \"Valgfri filer\",\n\t\"Downloaded\": \"Downloadet\",\n\t\"Download and help distribute all files\": \"Download og hjælp med at dele filer\",\n\t\"Total size\": \"Størrelse i alt\",\n\t\"Downloaded files\": \"Filer downloadet\",\n\n\t\"Database\": \"Database\",\n\t\"search feeds\": \"søgninger\",\n\t\"{feeds} query\": \"{feeds} søgninger\",\n\t\"Reload\": \"Genindlæs\",\n\t\"Rebuild\": \"Genopbyg\",\n\t\"No database found\": \"Ingen database fundet\",\n\n\t\"Identity address\": \"Autorisations ID\",\n\t\"Change\": \"Skift\",\n\n\t\"Update\": \"Opdater\",\n\t\"Pause\": \"Pause\",\n\t\"Resume\": \"Aktiv\",\n\t\"Delete\": \"Slet\",\n\t\"Are you sure?\": \"Er du sikker?\",\n\n\t\"Site address\": \"Side addresse\",\n\t\"Donate\": \"Doner penge\",\n\n\t\"Missing files\": \"Manglende filer\",\n\t\"{} try\": \"{} forsøg\",\n\t\"{} tries\": \"{} forsøg\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} mere\",\n\n\t\"This is my site\": \"Dette er min side\",\n\t\"Site title\": \"Side navn\",\n\t\"Site description\": \"Side beskrivelse\",\n\t\"Save site settings\": \"Gem side opsætning\",\n\n\t\"Content publishing\": \"Indhold offentliggøres\",\n\t\"Choose\": \"Vælg\",\n\t\"Sign\": \"Signer\",\n\t\"Publish\": \"Offentliggør\",\n\n\t\"This function is disabled on this proxy\": \"Denne funktion er slået fra på denne ZeroNet proxyEz a funkció ki van kapcsolva ezen a proxy-n\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 City database kunne ikke downloades: {}!
Download venligst databasen manuelt og udpak i data folder:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"GeoLite2 város adatbázis letöltése (csak egyszer kell, kb 20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 City database downloadet!\",\n\n\t\"Are you sure?\": \"Er du sikker?\",\n\t\"Site storage limit modified!\": \"Side max størrelse ændret!\",\n\t\"Database schema reloaded!\": \"Database definition genindlæst!\",\n\t\"Database rebuilding....\": \"Genopbygger database...\",\n\t\"Database rebuilt!\": \"Database genopbygget!\",\n\t\"Site updated!\": \"Side opdateret!\",\n\t\"Delete this site\": \"Slet denne side\",\n\t\"File write error: \": \"Fejl ved skrivning af fil: \",\n\t\"Site settings saved!\": \"Side opsætning gemt!\",\n\t\"Enter your private key:\": \"Indtast din private nøgle:\",\n\t\" Signed!\": \" Signeret!\",\n\t\"WebGL not supported\": \"WebGL er ikke supporteret\"\n}" }, { "path": "plugins/Sidebar/languages/de.json", "content": "{\n\t\"Peers\": \"Peers\",\n\t\"Connected\": \"Verbunden\",\n\t\"Connectable\": \"Verbindbar\",\n\t\"Connectable peers\": \"Verbindbare Peers\",\n\n\t\"Data transfer\": \"Datei Transfer\",\n\t\"Received\": \"Empfangen\",\n\t\"Received bytes\": \"Empfangene Bytes\",\n\t\"Sent\": \"Gesendet\",\n\t\"Sent bytes\": \"Gesendete Bytes\",\n\n\t\"Files\": \"Dateien\",\n\t\"Total\": \"Gesamt\",\n\t\"Image\": \"Bilder\",\n\t\"Other\": \"Sonstiges\",\n\t\"User data\": \"Nutzer Daten\",\n\n\t\"Size limit\": \"Speicher Limit\",\n\t\"limit used\": \"Limit benutzt\",\n\t\"free space\": \"freier Speicher\",\n\t\"Set\": \"Setzten\",\n\n\t\"Optional files\": \"Optionale Dateien\",\n\t\"Downloaded\": \"Heruntergeladen\",\n\t\"Download and help distribute all files\": \"Herunterladen und helfen alle Dateien zu verteilen\",\n\t\"Total size\": \"Gesamte Größe\",\n\t\"Downloaded files\": \"Heruntergeladene Dateien\",\n\n\t\"Database\": \"Datenbank\",\n\t\"search feeds\": \"Feeds durchsuchen\",\n\t\"{feeds} query\": \"{feeds} Abfrage\",\n\t\"Reload\": \"Neu laden\",\n\t\"Rebuild\": \"Neu bauen\",\n\t\"No database found\": \"Keine Datenbank gefunden\",\n\n\t\"Identity address\": \"Identitäts Adresse\",\n\t\"Change\": \"Ändern\",\n\n\t\"Update\": \"Aktualisieren\",\n\t\"Pause\": \"Pausieren\",\n\t\"Resume\": \"Fortsetzen\",\n\t\"Delete\": \"Löschen\",\n\t\"Are you sure?\": \"Bist du sicher?\",\n\n\t\"Site address\": \"Seiten Adresse\",\n\t\"Donate\": \"Spenden\",\n\n\t\"Missing files\": \"Fehlende Dateien\",\n\t\"{} try\": \"{} versuch\",\n\t\"{} tries\": \"{} versuche\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} mehr\",\n\n\t\"This is my site\": \"Das ist meine Seite\",\n\t\"Site title\": \"Seiten Titel\",\n\t\"Site description\": \"Seiten Beschreibung\",\n\t\"Save site settings\": \"Einstellungen der Seite speichern\",\n\n\t\"Content publishing\": \"Inhaltsveröffentlichung\",\n\t\"Choose\": \"Wähle\",\n\t\"Sign\": \"Signieren\",\n\t\"Publish\": \"Veröffentlichen\",\n\n\t\"This function is disabled on this proxy\": \"Diese Funktion ist auf dieser Proxy deaktiviert\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 City Datenbank Download Fehler: {}!
Bitte manuell herunterladen und die Datei in das Datei Verzeichnis extrahieren:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Herunterladen der GeoLite2 City Datenbank (einmalig, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 City Datenbank heruntergeladen!\",\n\n\t\"Are you sure?\": \"Bist du sicher?\",\n\t\"Site storage limit modified!\": \"Speicher Limit der Seite modifiziert!\",\n\t\"Database schema reloaded!\": \"Datebank Schema neu geladen!\",\n\t\"Database rebuilding....\": \"Datenbank neu bauen...\",\n\t\"Database rebuilt!\": \"Datenbank neu gebaut!\",\n\t\"Site updated!\": \"Seite aktualisiert!\",\n\t\"Delete this site\": \"Diese Seite löschen\",\n\t\"File write error: \": \"Datei schreib fehler:\",\n\t\"Site settings saved!\": \"Seiten Einstellungen gespeichert!\",\n\t\"Enter your private key:\": \"Gib deinen privaten Schlüssel ein:\",\n\t\" Signed!\": \" Signiert!\",\n\t\"WebGL not supported\": \"WebGL nicht unterstützt\"\n}\n" }, { "path": "plugins/Sidebar/languages/es.json", "content": "{\n\t\"Peers\": \"Pares\",\n\t\"Connected\": \"Conectados\",\n\t\"Connectable\": \"Conectables\",\n\t\"Connectable peers\": \"Pares conectables\",\n\n\t\"Data transfer\": \"Transferencia de datos\",\n\t\"Received\": \"Recibidos\",\n\t\"Received bytes\": \"Bytes recibidos\",\n\t\"Sent\": \"Enviados\",\n\t\"Sent bytes\": \"Bytes envidados\",\n\n\t\"Files\": \"Ficheros\",\n\t\"Total\": \"Total\",\n\t\"Image\": \"Imagen\",\n\t\"Other\": \"Otro\",\n\t\"User data\": \"Datos del usuario\",\n\n\t\"Size limit\": \"Límite de tamaño\",\n\t\"limit used\": \"Límite utilizado\",\n\t\"free space\": \"Espacio libre\",\n\t\"Set\": \"Establecer\",\n\n\t\"Optional files\": \"Ficheros opcionales\",\n\t\"Downloaded\": \"Descargado\",\n\t\"Download and help distribute all files\": \"Descargar y ayudar a distribuir todos los ficheros\",\n\t\"Total size\": \"Tamaño total\",\n\t\"Downloaded files\": \"Ficheros descargados\",\n\n\t\"Database\": \"Base de datos\",\n\t\"search feeds\": \"Fuentes de búsqueda\",\n\t\"{feeds} query\": \"{feeds} consulta\",\n\t\"Reload\": \"Recargar\",\n\t\"Rebuild\": \"Reconstruir\",\n\t\"No database found\": \"No se ha encontrado la base de datos\",\n\n\t\"Identity address\": \"Dirección de la identidad\",\n\t\"Change\": \"Cambiar\",\n\n\t\"Update\": \"Actualizar\",\n\t\"Pause\": \"Pausar\",\n\t\"Resume\": \"Reanudar\",\n\t\"Delete\": \"Borrar\",\n\n\t\"Site address\": \"Dirección del sitio\",\n\t\"Donate\": \"Donar\",\n\n\t\"Missing files\": \"Ficheros perdidos\",\n\t\"{} try\": \"{} intento\",\n\t\"{} tries\": \"{} intentos\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} más\",\n\n\t\"This is my site\": \"Este es mi sitio\",\n\t\"Site title\": \"Título del sitio\",\n\t\"Site description\": \"Descripción del sitio\",\n\t\"Save site settings\": \"Guardar la configuración del sitio\",\n\n\t\"Content publishing\": \"Publicación del contenido\",\n\t\"Choose\": \"Elegir\",\n\t\"Sign\": \"Firmar\",\n\t\"Publish\": \"Publicar\",\n\t\"This function is disabled on this proxy\": \"Esta función está deshabilitada en este proxy\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"¡Error de la base de datos GeoLite2： {}!
Por favor, descárgalo manualmente y descomprime al directorio de datos：
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Descargando la base de datos de GeoLite2 (una única vez, ~20MB）...\",\n\t\"GeoLite2 City database downloaded!\": \"¡Base de datos de GeoLite2 descargada!\",\n\n\t\"Are you sure?\": \"¿Estás seguro?\",\n\t\"Site storage limit modified!\": \"¡Límite de almacenamiento del sitio modificado!\",\n\t\"Database schema reloaded!\": \"¡Esquema de la base de datos recargado!\",\n\t\"Database rebuilding....\": \"Reconstruyendo la base de datos...\",\n\t\"Database rebuilt!\": \"¡Base de datos reconstruida!\",\n\t\"Site updated!\": \"¡Sitio actualizado!\",\n\t\"Delete this site\": \"Borrar este sitio\",\n\t\"File write error: \": \"Error de escritura de fichero：\",\n\t\"Site settings saved!\": \"¡Configuración del sitio guardada!\",\n\t\"Enter your private key:\": \"Introduce tu clave privada：\",\n\t\" Signed!\": \" ¡firmado!\",\n\t\"WebGL not supported\": \"WebGL no está soportado\"\n}\n" }, { "path": "plugins/Sidebar/languages/fr.json", "content": "{\n\t\"Peers\": \"Pairs\",\n\t\"Connected\": \"Connectés\",\n\t\"Connectable\": \"Accessibles\",\n\t\"Connectable peers\": \"Pairs accessibles\",\n\n\t\"Data transfer\": \"Données transférées\",\n\t\"Received\": \"Reçues\",\n\t\"Received bytes\": \"Bytes reçus\",\n\t\"Sent\": \"Envoyées\",\n\t\"Sent bytes\": \"Bytes envoyés\",\n\n\t\"Files\": \"Fichiers\",\n\t\"Total\": \"Total\",\n\t\"Image\": \"Image\",\n\t\"Other\": \"Autre\",\n\t\"User data\": \"Utilisateurs\",\n\n\t\"Size limit\": \"Taille maximale\",\n\t\"limit used\": \"utlisé\",\n\t\"free space\": \"libre\",\n\t\"Set\": \"Modifier\",\n\n\t\"Optional files\": \"Fichiers optionnels\",\n\t\"Downloaded\": \"Téléchargé\",\n\t\"Download and help distribute all files\": \"Télécharger et distribuer tous les fichiers\",\n\t\"Total size\": \"Taille totale\",\n\t\"Downloaded files\": \"Fichiers téléchargés\",\n\n\t\"Database\": \"Base de données\",\n\t\"search feeds\": \"recherche\",\n\t\"{feeds} query\": \"{feeds} requête\",\n\t\"Reload\": \"Recharger\",\n\t\"Rebuild\": \"Reconstruire\",\n\t\"No database found\": \"Aucune base de données trouvée\",\n\n\t\"Identity address\": \"Adresse d'identité\",\n\t\"Change\": \"Modifier\",\n\n\t\"Site control\": \"Opérations\",\n\t\"Update\": \"Mettre à jour\",\n\t\"Pause\": \"Suspendre\",\n\t\"Resume\": \"Reprendre\",\n\t\"Delete\": \"Supprimer\",\n\t\"Are you sure?\": \"Êtes-vous certain?\",\n\n\t\"Site address\": \"Adresse du site\",\n\t\"Donate\": \"Faire un don\",\n\n\t\"Missing files\": \"Fichiers manquants\",\n\t\"{} try\": \"{} essai\",\n\t\"{} tries\": \"{} essais\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} manquants\",\n\n\t\"This is my site\": \"Ce site m'appartient\",\n\t\"Site title\": \"Nom du site\",\n\t\"Site description\": \"Description du site\",\n\t\"Save site settings\": \"Enregistrer les paramètres\",\n\n\t\"Content publishing\": \"Publication du contenu\",\n\t\"Choose\": \"Sélectionner\",\n\t\"Sign\": \"Signer\",\n\t\"Publish\": \"Publier\",\n\n\t\"This function is disabled on this proxy\": \"Cette fonction est désactivé sur ce proxy\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"Erreur au téléchargement de la base de données GeoLite2: {}!
Téléchargez et décompressez dans le dossier data:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Téléchargement de la base de données GeoLite2 (une seule fois, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"Base de données GeoLite2 téléchargée!\",\n\n\t\"Are you sure?\": \"Êtes-vous certain?\",\n\t\"Site storage limit modified!\": \"Taille maximale modifiée!\",\n\t\"Database schema reloaded!\": \"Base de données rechargée!\",\n\t\"Database rebuilding....\": \"Reconstruction de la base de données...\",\n\t\"Database rebuilt!\": \"Base de données reconstruite!\",\n\t\"Site updated!\": \"Site mis à jour!\",\n\t\"Delete this site\": \"Supprimer ce site\",\n\t\"File write error: \": \"Erreur à l'écriture du fichier: \",\n\t\"Site settings saved!\": \"Paramètres du site enregistrés!\",\n\t\"Enter your private key:\": \"Entrez votre clé privée:\",\n\t\" Signed!\": \" Signé!\",\n\t\"WebGL not supported\": \"WebGL n'est pas supporté\"\n}\n" }, { "path": "plugins/Sidebar/languages/hu.json", "content": "{\n\t\"Peers\": \"Csatlakozási pontok\",\n\t\"Connected\": \"Csaltakozva\",\n\t\"Connectable\": \"Csatlakozható\",\n\t\"Connectable peers\": \"Csatlakozható peer-ek\",\n\n\t\"Data transfer\": \"Adatátvitel\",\n\t\"Received\": \"Fogadott\",\n\t\"Received bytes\": \"Fogadott byte-ok\",\n\t\"Sent\": \"Küldött\",\n\t\"Sent bytes\": \"Küldött byte-ok\",\n\n\t\"Files\": \"Fájlok\",\n\t\"Total\": \"Összesen\",\n\t\"Image\": \"Kép\",\n\t\"Other\": \"Egyéb\",\n\t\"User data\": \"Felh. adat\",\n\n\t\"Size limit\": \"Méret korlát\",\n\t\"limit used\": \"felhasznált\",\n\t\"free space\": \"szabad hely\",\n\t\"Set\": \"Beállít\",\n\n\t\"Optional files\": \"Opcionális fájlok\",\n\t\"Downloaded\": \"Letöltött\",\n\t\"Download and help distribute all files\": \"Minden opcionális fájl letöltése\",\n\t\"Total size\": \"Teljes méret\",\n\t\"Downloaded files\": \"Letöltve\",\n\n\t\"Database\": \"Adatbázis\",\n\t\"search feeds\": \"Keresés források\",\n\t\"{feeds} query\": \"{feeds} lekérdezés\",\n\t\"Reload\": \"Újratöltés\",\n\t\"Rebuild\": \"Újraépítés\",\n\t\"No database found\": \"Adatbázis nem található\",\n\n\t\"Identity address\": \"Azonosító cím\",\n\t\"Change\": \"Módosít\",\n\n\t\"Site control\": \"Oldal műveletek\",\n\t\"Update\": \"Frissít\",\n\t\"Pause\": \"Szünteltet\",\n\t\"Resume\": \"Folytat\",\n\t\"Delete\": \"Töröl\",\n\t\"Are you sure?\": \"Biztos vagy benne?\",\n\n\t\"Site address\": \"Oldal címe\",\n\t\"Donate\": \"Támogatás\",\n\n\t\"Missing files\": \"Hiányzó fájlok\",\n\t\"{} try\": \"{} próbálkozás\",\n\t\"{} tries\": \"{} próbálkozás\",\n\t\"+ {num_bad_files} more\": \"+ még {num_bad_files} darab\",\n\n\t\"This is my site\": \"Ez az én oldalam\",\n\t\"Site title\": \"Oldal neve\",\n\t\"Site description\": \"Oldal leírása\",\n\t\"Save site settings\": \"Oldal beállítások mentése\",\n\n\t\"Content publishing\": \"Tartalom publikálás\",\n\t\"Choose\": \"Válassz\",\n\t\"Sign\": \"Aláírás\",\n\t\"Publish\": \"Publikálás\",\n\n\t\"This function is disabled on this proxy\": \"Ez a funkció ki van kapcsolva ezen a proxy-n\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 város adatbázis letöltési hiba: {}!
A térképhez töltsd le és csomagold ki a data könyvtárba:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"GeoLite2 város adatbázis letöltése (csak egyszer kell, kb 20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 város adatbázis letöltve!\",\n\n\t\"Are you sure?\": \"Biztos vagy benne?\",\n\t\"Site storage limit modified!\": \"Az oldalt méret korlát módosítva!\",\n\t\"Database schema reloaded!\": \"Adatbázis séma újratöltve!\",\n\t\"Database rebuilding....\": \"Adatbázis újraépítés...\",\n\t\"Database rebuilt!\": \"Adatbázis újraépítve!\",\n\t\"Site updated!\": \"Az oldal frissítve!\",\n\t\"Delete this site\": \"Az oldal törlése\",\n\t\"File write error: \": \"Fájl írási hiba: \",\n\t\"Site settings saved!\": \"Az oldal beállításai elmentve!\",\n\t\"Enter your private key:\": \"Add meg a privát kulcsod:\",\n\t\" Signed!\": \" Aláírva!\",\n\t\"WebGL not supported\": \"WebGL nem támogatott\"\n}\n" }, { "path": "plugins/Sidebar/languages/it.json", "content": "{\n\t\"Peers\": \"Peer\",\n\t\"Connected\": \"Connessi\",\n\t\"Connectable\": \"Collegabili\",\n\t\"Connectable peers\": \"Peer collegabili\",\n\n\t\"Data transfer\": \"Trasferimento dati\",\n\t\"Received\": \"Ricevuti\",\n\t\"Received bytes\": \"Byte ricevuti\",\n\t\"Sent\": \"Inviati\",\n\t\"Sent bytes\": \"Byte inviati\",\n\n\t\"Files\": \"File\",\n\t\"Total\": \"Totale\",\n\t\"Image\": \"Imagine\",\n\t\"Other\": \"Altro\",\n\t\"User data\": \"Dati utente\",\n\n\t\"Size limit\": \"Limite dimensione\",\n\t\"limit used\": \"limite usato\",\n\t\"free space\": \"spazio libero\",\n\t\"Set\": \"Imposta\",\n\n\t\"Optional files\": \"File facoltativi\",\n\t\"Downloaded\": \"Scaricati\",\n\t\"Download and help distribute all files\": \"Scarica e aiuta a distribuire tutti i file\",\n\t\"Total size\": \"Dimensione totale\",\n\t\"Downloaded files\": \"File scaricati\",\n\n\t\"Database\": \"Database\",\n\t\"search feeds\": \"ricerca di feed\",\n\t\"{feeds} query\": \"{feeds} interrogazione\",\n\t\"Reload\": \"Ricaricare\",\n\t\"Rebuild\": \"Ricostruire\",\n\t\"No database found\": \"Nessun database trovato\",\n\n\t\"Identity address\": \"Indirizzo di identità\",\n\t\"Change\": \"Cambia\",\n\n\t\"Update\": \"Aggiorna\",\n\t\"Pause\": \"Sospendi\",\n\t\"Resume\": \"Riprendi\",\n\t\"Delete\": \"Cancella\",\n\t\"Are you sure?\": \"Sei sicuro?\",\n\n\t\"Site address\": \"Indirizzo sito\",\n\t\"Donate\": \"Dona\",\n\n\t\"Missing files\": \"File mancanti\",\n\t\"{} try\": \"{} tenta\",\n\t\"{} tries\": \"{} prova\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} altri\",\n\n\t\"This is my site\": \"Questo è il mio sito\",\n\t\"Site title\": \"Titolo sito\",\n\t\"Site description\": \"Descrizione sito\",\n\t\"Save site settings\": \"Salva impostazioni sito\",\n\n\t\"Content publishing\": \"Pubblicazione contenuto\",\n\t\"Choose\": \"Scegli\",\n\t\"Sign\": \"Firma\",\n\t\"Publish\": \"Pubblica\",\n\n\t\"This function is disabled on this proxy\": \"Questa funzione è disabilitata su questo proxy\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"Errore scaricamento database GeoLite2 City: {}!
Si prega di scaricarlo manualmente e spacchetarlo nella cartella dir:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Scaricamento database GeoLite2 City (solo una volta, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"Database GeoLite2 City scaricato!\",\n\n\t\"Are you sure?\": \"Sei sicuro?\",\n\t\"Site storage limit modified!\": \"Limite di archiviazione del sito modificato!\",\n\t\"Database schema reloaded!\": \"Schema database ricaricato!\",\n\t\"Database rebuilding....\": \"Ricostruzione database...\",\n\t\"Database rebuilt!\": \"Database ricostruito!\",\n\t\"Site updated!\": \"Sito aggiornato!\",\n\t\"Delete this site\": \"Cancella questo sito\",\n\t\"File write error: \": \"Errore scrittura file:\",\n\t\"Site settings saved!\": \"Impostazioni sito salvate!\",\n\t\"Enter your private key:\": \"Inserisci la tua chiave privata:\",\n\t\" Signed!\": \" Firmato!\",\n\t\"WebGL not supported\": \"WebGL non supportato\"\n}\n" }, { "path": "plugins/Sidebar/languages/jp.json", "content": "{\n\t\"Copy to clipboard\": \"クリップボードにコピー\",\n\t\"Peers\": \"ピア\",\n\t\"Connected\": \"接続済み\",\n\t\"Connectable\": \"利用可能\",\n\t\"Connectable peers\": \"ピアに接続可能\",\n\t\"Onion\": \"Onion\",\n\t\"Local\": \"ローカル\",\n\n\t\"Data transfer\": \"データ転送\",\n\t\"Received\": \"受信\",\n\t\"Received bytes\": \"受信バイト数\",\n\t\"Sent\": \"送信\",\n\t\"Sent bytes\": \"送信バイト数\",\n\n\t\"Files\": \"ファイル\",\n\t\"Browse files\": \"ファイルを見る\",\n\t\"Save as .zip\": \"ZIP形式で保存\",\n\t\"Total\": \"合計\",\n\t\"Image\": \"画像\",\n\t\"Other\": \"その他\",\n\t\"User data\": \"ユーザーデータ\",\n\n\t\"Size limit\": \"サイズ制限\",\n\t\"limit used\": \"使用上限\",\n\t\"free space\": \"フリースペース\",\n\t\"Set\": \"セット\",\n\n\t\"Optional files\": \"オプション ファイル\",\n\t\"Downloaded\": \"ダウンロード済み\",\n\t\"Help distribute added optional files\": \"オプションファイルの配布を支援する\",\n\t\"Auto download big file size limit\": \"大きなファイルの自動ダウンロードのサイズ制限\",\n\t\"Download previous files\": \"以前のファイルのダウンロード\",\n\t\"Optional files download started\": \"オプションファイルのダウンロードを開始\",\n\t\"Optional files downloaded\": \"オプションファイルのダウンロードが完了しました\",\n\t\"Download and help distribute all files\": \"ダウンロードしてすべてのファイルの配布を支援する\",\n\t\"Total size\": \"合計サイズ\",\n\t\"Downloaded files\": \"ダウンロードされたファイル\",\n\n\t\"Database\": \"データベース\",\n\t\"search feeds\": \"フィードを検索する\",\n\t\"{feeds} query\": \"{feeds} お問い合わせ\",\n\t\"Reload\": \"再読込\",\n\t\"Rebuild\": \"再ビルド\",\n\t\"No database found\": \"データベースが見つかりません\",\n\n\t\"Identity address\": \"あなたの識別アドレス\",\n\t\"Change\": \"編集\",\n\n\t\"Site control\": \"サイト管理\",\n\t\"Update\": \"更新\",\n\t\"Pause\": \"一時停止\",\n\t\"Resume\": \"再開\",\n\t\"Delete\": \"削除\",\n\t\"Are you sure?\": \"本当によろしいですか？\",\n\n\t\"Site address\": \"サイトアドレス\",\n\t\"Donate\": \"寄付する\",\n\n\t\"Missing files\": \"ファイルがありません\",\n\t\"{} try\": \"{} 試す\",\n\t\"{} tries\": \"{} 試行\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} more\",\n\n\t\"This is my site\": \"これは私のサイトです\",\n\t\"Site title\": \"サイトタイトル\",\n\t\"Site description\": \"サイトの説明\",\n\t\"Save site settings\": \"サイトの設定を保存する\",\n\t\"Open site directory\": \"サイトのディレクトリを開く\",\n\n\t\"Content publishing\": \"コンテンツを公開する\",\n\t\"Add saved private key\": \"秘密鍵の追加と保存\",\n\t\"Save\": \"保存\",\n\t\"Private key saved.\": \"秘密鍵が保存されています\",\n\t\"Private key saved for site signing\": \"サイトに署名するための秘密鍵を保存\",\n\t\"Forgot\": \"わすれる\",\n\t\"Saved private key removed\": \"保存された秘密鍵を削除しました\",\n\t\"Choose\": \"選択\",\n\t\"Sign\": \"署名\",\n\t\"Publish\": \"公開する\",\n\t\"Sign and publish\": \"署名して公開\",\n\n\t\"This function is disabled on this proxy\": \"この機能はこのプロキシで無効になっています\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 Cityデータベースのダウンロードエラー: {}!
手動でダウンロードして、フォルダに解凍してください。:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"GeoLite2 Cityデータベースの読み込み (これは一度だけ行われます, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 Cityデータベースがダウンロードされました！\",\n\n\t\"Are you sure?\": \"本当によろしいですか？\",\n\t\"Site storage limit modified!\": \"サイトの保存容量の制限が変更されました！\",\n\t\"Database schema reloaded!\": \"データベーススキーマがリロードされました！\",\n\t\"Database rebuilding....\": \"データベースの再構築中....\",\n\t\"Database rebuilt!\": \"データベースが再構築されました！\",\n\t\"Site updated!\": \"サイトが更新されました！\",\n\t\"Delete this site\": \"このサイトを削除する\",\n\t\"Blacklist\": \"NG\",\n\t\"Blacklist this site\": \"NGリストに入れる\",\n\t\"Reason\": \"理由\",\n\t\"Delete and Blacklist\": \"削除してNG\",\n\t\"File write error: \": \"ファイル書き込みエラー:\",\n\t\"Site settings saved!\": \"サイト設定が保存されました！\",\n\t\"Enter your private key:\": \"秘密鍵を入力してください:\",\n\t\" Signed!\": \" 署名しました!\",\n\t\"WebGL not supported\": \"WebGLはサポートされていません\"\n}\n" }, { "path": "plugins/Sidebar/languages/pl.json", "content": "{\n\t\"Peers\": \"Użytkownicy równorzędni\",\n\t\"Connected\": \"Połączony\",\n\t\"Connectable\": \"Możliwy do podłączenia\",\n\t\"Connectable peers\": \"Połączeni użytkownicy równorzędni\",\n\n\t\"Data transfer\": \"Transfer danych\",\n\t\"Received\": \"Odebrane\",\n\t\"Received bytes\": \"Odebrany bajty\",\n\t\"Sent\": \"Wysłane\",\n\t\"Sent bytes\": \"Wysłane bajty\",\n\n\t\"Files\": \"Pliki\",\n\t\"Total\": \"Sumarycznie\",\n\t\"Image\": \"Obraz\",\n\t\"Other\": \"Inne\",\n\t\"User data\": \"Dane użytkownika\",\n\n\t\"Size limit\": \"Rozmiar limitu\",\n\t\"limit used\": \"zużyty limit\",\n\t\"free space\": \"wolna przestrzeń\",\n\t\"Set\": \"Ustaw\",\n\n\t\"Optional files\": \"Pliki opcjonalne\",\n\t\"Downloaded\": \"Ściągnięte\",\n\t\"Download and help distribute all files\": \"Ściągnij i pomóż rozpowszechniać wszystkie pliki\",\n\t\"Total size\": \"Rozmiar sumaryczny\",\n\t\"Downloaded files\": \"Ściągnięte pliki\",\n\n\t\"Database\": \"Baza danych\",\n\t\"search feeds\": \"przeszukaj zasoby\",\n\t\"{feeds} query\": \"{feeds} pytanie\",\n\t\"Reload\": \"Odśwież\",\n\t\"Rebuild\": \"Odbuduj\",\n\t\"No database found\": \"Nie odnaleziono bazy danych\",\n\n\t\"Identity address\": \"Adres identyfikacyjny\",\n\t\"Change\": \"Zmień\",\n\n\t\"Site control\": \"Kontrola strony\",\n\t\"Update\": \"Zaktualizuj\",\n\t\"Pause\": \"Wstrzymaj\",\n\t\"Resume\": \"Wznów\",\n\t\"Delete\": \"Skasuj\",\n\t\"Are you sure?\": \"Jesteś pewien?\",\n\n\t\"Site address\": \"Adres strony\",\n\t\"Donate\": \"Wspomóż\",\n\n\t\"Missing files\": \"Brakujące pliki\",\n\t\"{} try\": \"{} próba\",\n\t\"{} tries\": \"{} próby\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} więcej\",\n\n\t\"This is my site\": \"To moja strona\",\n\t\"Site title\": \"Tytuł strony\",\n\t\"Site description\": \"Opis strony\",\n\t\"Save site settings\": \"Zapisz ustawienia strony\",\n\n\t\"Content publishing\": \"Publikowanie treści\",\n\t\"Choose\": \"Wybierz\",\n\t\"Sign\": \"Podpisz\",\n\t\"Publish\": \"Opublikuj\",\n\n\t\"This function is disabled on this proxy\": \"Ta funkcja jest zablokowana w tym proxy\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"Błąd ściągania bazy danych GeoLite2 City: {}!
Proszę ściągnąć ją recznie i wypakować do katalogu danych:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Ściąganie bazy danych GeoLite2 City (tylko jednorazowo, ok. 20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"Baza danych GeoLite2 City ściagnięta!\",\n\n\t\"Are you sure?\": \"Jesteś pewien?\",\n\t\"Site storage limit modified!\": \"Limit pamięci strony zmodyfikowany!\",\n\t\"Database schema reloaded!\": \"Schemat bazy danych załadowany ponownie!\",\n\t\"Database rebuilding....\": \"Przebudowywanie bazy danych...\",\n\t\"Database rebuilt!\": \"Baza danych przebudowana!\",\n\t\"Site updated!\": \"Strona zaktualizowana!\",\n\t\"Delete this site\": \"Usuń tę stronę\",\n\t\"File write error: \": \"Błąd zapisu pliku: \",\n\t\"Site settings saved!\": \"Ustawienia strony zapisane!\",\n\t\"Enter your private key:\": \"Wpisz swój prywatny klucz:\",\n\t\" Signed!\": \" Podpisane!\",\n\t\"WebGL not supported\": \"WebGL nie jest obsługiwany\"\n}\n" }, { "path": "plugins/Sidebar/languages/pt-br.json", "content": "{\n\t\"Copy to clipboard\": \"Copiar para área de transferência (clipboard)\",\n\t\"Peers\": \"Peers\",\n\t\"Connected\": \"Ligados\",\n\t\"Connectable\": \"Disponíveis\",\n\t\"Onion\": \"Onion\",\n\t\"Local\": \"Locais\",\n\t\"Connectable peers\": \"Peers disponíveis\",\n\n\t\"Data transfer\": \"Transferência de dados\",\n\t\"Received\": \"Recebidos\",\n\t\"Received bytes\": \"Bytes recebidos\",\n\t\"Sent\": \"Enviados\",\n\t\"Sent bytes\": \"Bytes enviados\",\n\n\t\"Files\": \"Arquivos\",\n\t\"Save as .zip\": \"Salvar como .zip\",\n\t\"Total\": \"Total\",\n\t\"Image\": \"Imagem\",\n\t\"Other\": \"Outros\",\n\t\"User data\": \"Dados do usuário\",\n\n\t\"Size limit\": \"Limite de tamanho\",\n\t\"limit used\": \"limite utilizado\",\n\t\"free space\": \"espaço livre\",\n\t\"Set\": \"Definir\",\n\n\t\"Optional files\": \"Arquivos opcionais\",\n\t\"Downloaded\": \"Baixados\",\n\t\"Download and help distribute all files\": \"Baixar e ajudar a distribuir todos os arquivos\",\n\t\"Total size\": \"Tamanho total\",\n\t\"Downloaded files\": \"Arquivos baixados\",\n\n\t\"Database\": \"Banco de dados\",\n\t\"search feeds\": \"pesquisar feeds\",\n\t\"{feeds} query\": \"consulta de {feeds}\",\n\t\"Reload\": \"Recarregar\",\n\t\"Rebuild\": \"Reconstruir\",\n\t\"No database found\": \"Base de dados não encontrada\",\n\n\t\"Identity address\": \"Endereço de identidade\",\n\t\"Change\": \"Alterar\",\n\n\t\"Site control\": \"Controle do site\",\n\t\"Update\": \"Atualizar\",\n\t\"Pause\": \"Suspender\",\n\t\"Resume\": \"Continuar\",\n\t\"Delete\": \"Remover\",\n\t\"Are you sure?\": \"Tem certeza?\",\n\n\t\"Site address\": \"Endereço do site\",\n\t\"Donate\": \"Doar\",\n\n\t\"Needs to be updated\": \"Necessitam ser atualizados\",\n\t\"{} try\": \"{} tentativa\",\n\t\"{} tries\": \"{} tentativas\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} adicionais\",\n\n\t\"This is my site\": \"Este é o meu site\",\n\t\"Site title\": \"Título do site\",\n\t\"Site description\": \"Descrição do site\",\n\t\"Save site settings\": \"Salvar definições do site\",\n\t\"Open site directory\": \"Abrir diretório do site\",\n\n\t\"Content publishing\": \"Publicação do conteúdo\",\n\t\"Choose\": \"Escolher\",\n\t\"Sign\": \"Assinar\",\n\t\"Publish\": \"Publicar\",\n\t\"Sign and publish\": \"Assinar e publicar\",\n\t\"add saved private key\": \"adicionar privatekey (chave privada) para salvar\",\n\t\"Private key saved for site signing\": \"Privatekey foi salva para assinar o site\",\n\t\"Private key saved.\": \"Privatekey salva.\",\n\t\"forgot\": \"esquecer\",\n\t\"Saved private key removed\": \"Privatekey salva foi removida\",\n\t\"This function is disabled on this proxy\": \"Esta função encontra-se desativada neste proxy\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"Erro ao baixar a base de dados GeoLite2 City: {}!
Por favor baixe manualmente e descompacte os dados para a seguinte pasta:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Baixando a base de dados GeoLite2 City (uma única vez, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"A base de dados GeoLite2 City foi baixada!\",\n\n\t\"Are you sure?\": \"Tem certeza?\",\n\t\"Site storage limit modified!\": \"O limite de armazenamento do site foi modificado!\",\n\t\"Database schema reloaded!\": \"O esquema da base de dados foi atualizado!\",\n\t\"Database rebuilding....\": \"Reconstruindo base de dados...\",\n\t\"Database rebuilt!\": \"Base de dados reconstruída!\",\n\t\"Site updated!\": \"Site atualizado!\",\n\t\"Delete this site\": \"Remover este site\",\n\t\"Blacklist\": \"Blacklist\",\n\t\"Blacklist this site\": \"Blacklistar este site\",\n\t\"Reason\": \"Motivo\",\n\t\"Delete and Blacklist\": \"Deletar e blacklistar\",\n\t\"File write error: \": \"Erro de escrita de arquivo: \",\n\t\"Site settings saved!\": \"Definições do site salvas!\",\n\t\"Enter your private key:\": \"Digite sua chave privada:\",\n\t\" Signed!\": \" Assinado!\",\n\t\"WebGL not supported\": \"WebGL não é suportado\",\n\t\"Save as .zip\": \"Salvar como .zip\"\n}\n" }, { "path": "plugins/Sidebar/languages/ru.json", "content": "{\n\t\"Peers\": \"Пиры\",\n\t\"Connected\": \"Подключенные\",\n\t\"Connectable\": \"Доступные\",\n\t\"Connectable peers\": \"Пиры доступны для подключения\",\n\n\t\"Data transfer\": \"Передача данных\",\n\t\"Received\": \"Получено\",\n\t\"Received bytes\": \"Получено байн\",\n\t\"Sent\": \"Отправлено\",\n\t\"Sent bytes\": \"Отправлено байт\",\n\n\t\"Files\": \"Файлы\",\n\t\"Total\": \"Всего\",\n\t\"Image\": \"Изображений\",\n\t\"Other\": \"Другое\",\n\t\"User data\": \"Ваш контент\",\n\n\t\"Size limit\": \"Ограничение по размеру\",\n\t\"limit used\": \"Использовано\",\n\t\"free space\": \"Доступно\",\n\t\"Set\": \"Установить\",\n\n\t\"Optional files\": \"Опциональные файлы\",\n\t\"Downloaded\": \"Загружено\",\n\t\"Download and help distribute all files\": \"Загрузить опциональные файлы для помощи сайту\",\n\t\"Total size\": \"Объём\",\n\t\"Downloaded files\": \"Загруженные файлы\",\n\n\t\"Database\": \"База данных\",\n\t\"search feeds\": \"поиск подписок\",\n\t\"{feeds} query\": \"{feeds} запрос\",\n\t\"Reload\": \"Перезагрузить\",\n\t\"Rebuild\": \"Перестроить\",\n\t\"No database found\": \"База данных не найдена\",\n\n\t\"Identity address\": \"Уникальный адрес\",\n\t\"Change\": \"Изменить\",\n\n\t\"Site control\": \"Управление сайтом\",\n\t\"Update\": \"Обновить\",\n\t\"Pause\": \"Пауза\",\n\t\"Resume\": \"Продолжить\",\n\t\"Delete\": \"Удалить\",\n\t\"Are you sure?\": \"Вы уверены?\",\n\n\t\"Site address\": \"Адрес сайта\",\n\t\"Donate\": \"Пожертвовать\",\n\n\t\"Missing files\": \"Отсутствующие файлы\",\n\t\"{} try\": \"{} попробовать\",\n\t\"{} tries\": \"{} попыток\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} ещё\",\n\n\t\"This is my site\": \"Это мой сайт\",\n\t\"Site title\": \"Название сайта\",\n\t\"Site description\": \"Описание сайта\",\n\t\"Save site settings\": \"Сохранить настройки сайта\",\n\n\t\"Content publishing\": \"Публикация контента\",\n\t\"Choose\": \"Выбрать\",\n\t\"Sign\": \"Подписать\",\n\t\"Publish\": \"Опубликовать\",\n\n\t\"This function is disabled on this proxy\": \"Эта функция отключена на этом прокси\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"Ошибка загрузки базы городов GeoLite2: {}!
Пожалуйста, загрузите её вручную и распакуйте в папку:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"Загрузка базы городов GeoLite2 (это делается только 1 раз, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"База GeoLite2 успешно загружена!\",\n\n\t\"Are you sure?\": \"Вы уверены?\",\n\t\"Site storage limit modified!\": \"Лимит хранилища для сайта изменен!\",\n\t\"Database schema reloaded!\": \"Схема базы данных перезагружена!\",\n\t\"Database rebuilding....\": \"Перестройка базы данных...\",\n\t\"Database rebuilt!\": \"База данных перестроена!\",\n\t\"Site updated!\": \"Сайт обновлён!\",\n\t\"Delete this site\": \"Удалить этот сайт\",\n\t\"File write error: \": \"Ошибка записи файла:\",\n\t\"Site settings saved!\": \"Настройки сайта сохранены!\",\n\t\"Enter your private key:\": \"Введите свой приватный ключ:\",\n\t\" Signed!\": \" Подписано!\",\n\t\"WebGL not supported\": \"WebGL не поддерживается\"\n}\n" }, { "path": "plugins/Sidebar/languages/tr.json", "content": "{\n\t\"Peers\": \"Eşler\",\n\t\"Connected\": \"Bağlı\",\n\t\"Connectable\": \"Erişilebilir\",\n\t\"Connectable peers\": \"Bağlanılabilir eşler\",\n\n\t\"Data transfer\": \"Veri aktarımı\",\n\t\"Received\": \"Alınan\",\n\t\"Received bytes\": \"Bayt alındı\",\n\t\"Sent\": \"Gönderilen\",\n\t\"Sent bytes\": \"Bayt gönderildi\",\n\n\t\"Files\": \"Dosyalar\",\n\t\"Total\": \"Toplam\",\n\t\"Image\": \"Resim\",\n\t\"Other\": \"Diğer\",\n\t\"User data\": \"Kullanıcı verisi\",\n\n\t\"Size limit\": \"Boyut sınırı\",\n\t\"limit used\": \"kullanılan\",\n\t\"free space\": \"boş\",\n\t\"Set\": \"Ayarla\",\n\n\t\"Optional files\": \"İsteğe bağlı dosyalar\",\n\t\"Downloaded\": \"İndirilen\",\n\t\"Download and help distribute all files\": \"Tüm dosyaları indir ve yayılmalarına yardım et\",\n\t\"Total size\": \"Toplam boyut\",\n\t\"Downloaded files\": \"İndirilen dosyalar\",\n\n\t\"Database\": \"Veritabanı\",\n\t\"search feeds\": \"kaynak ara\",\n\t\"{feeds} query\": \"{feeds} sorgu\",\n\t\"Reload\": \"Yenile\",\n\t\"Rebuild\": \"Yapılandır\",\n\t\"No database found\": \"Veritabanı yok\",\n\n\t\"Identity address\": \"Kimlik adresi\",\n\t\"Change\": \"Değiştir\",\n\n\t\"Site control\": \"Site kontrolü\",\n\t\"Update\": \"Güncelle\",\n\t\"Pause\": \"Duraklat\",\n\t\"Resume\": \"Sürdür\",\n\t\"Delete\": \"Sil\",\n\t\"Are you sure?\": \"Emin misin?\",\n\n\t\"Site address\": \"Site adresi\",\n\t\"Donate\": \"Bağış yap\",\n\n\t\"Missing files\": \"Eksik dosyalar\",\n\t\"{} try\": \"{} deneme\",\n\t\"{} tries\": \"{} deneme\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} tane daha\",\n\n\t\"This is my site\": \"Bu benim sitem\",\n\t\"Site title\": \"Site başlığı\",\n\t\"Site description\": \"Site açıklaması\",\n\t\"Save site settings\": \"Site ayarlarını kaydet\",\n\n\t\"Content publishing\": \"İçerik yayımlanıyor\",\n\t\"Choose\": \"Seç\",\n\t\"Sign\": \"İmzala\",\n\t\"Publish\": \"Yayımla\",\n\n\t\"This function is disabled on this proxy\": \"Bu özellik bu vekilde kullanılamaz\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 Şehir veritabanı indirme hatası: {}!
Lütfen kendiniz indirip aşağıdaki konuma açınınız:
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"GeoLite2 Şehir veritabanı indiriliyor (sadece bir kere, ~20MB)...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 Şehir veritabanı indirildi!\",\n\n\t\"Are you sure?\": \"Emin misiniz?\",\n\t\"Site storage limit modified!\": \"Site saklama sınırı değiştirildi!\",\n\t\"Database schema reloaded!\": \"Veritabanı şeması yeniden yüklendi!\",\n\t\"Database rebuilding....\": \"Veritabanı yeniden inşa ediliyor...\",\n\t\"Database rebuilt!\": \"Veritabanı yeniden inşa edildi!\",\n\t\"Site updated!\": \"Site güncellendi!\",\n\t\"Delete this site\": \"Bu siteyi sil\",\n\t\"File write error: \": \"Dosya yazma hatası: \",\n\t\"Site settings saved!\": \"Site ayarları kaydedildi!\",\n\t\"Enter your private key:\": \"Özel anahtarınızı giriniz:\",\n\t\" Signed!\": \" İmzala!\",\n\t\"WebGL not supported\": \"WebGL desteklenmiyor\"\n}\n" }, { "path": "plugins/Sidebar/languages/zh-tw.json", "content": "{\n\t\"Peers\": \"節點數\",\n\t\"Connected\": \"已連線\",\n\t\"Connectable\": \"可連線\",\n\t\"Connectable peers\": \"可連線節點\",\n\n\t\"Data transfer\": \"數據傳輸\",\n\t\"Received\": \"已接收\",\n\t\"Received bytes\": \"已接收位元組\",\n\t\"Sent\": \"已傳送\",\n\t\"Sent bytes\": \"已傳送位元組\",\n\n\t\"Files\": \"檔案\",\n\t\"Total\": \"共計\",\n\t\"Image\": \"圖片\",\n\t\"Other\": \"其他\",\n\t\"User data\": \"使用者數據\",\n\n\t\"Size limit\": \"大小限制\",\n\t\"limit used\": \"已使用\",\n\t\"free space\": \"可用空間\",\n\t\"Set\": \"偏好設定\",\n\n\t\"Optional files\": \"可選檔案\",\n\t\"Downloaded\": \"已下載\",\n\t\"Download and help distribute all files\": \"下載並幫助分發所有檔案\",\n\t\"Total size\": \"總大小\",\n\t\"Downloaded files\": \"下載的檔案\",\n\n\t\"Database\": \"資料庫\",\n\t\"search feeds\": \"搜尋供稿\",\n\t\"{feeds} query\": \"{feeds} 查詢 \",\n\t\"Reload\": \"重新整理\",\n\t\"Rebuild\": \"重建\",\n\t\"No database found\": \"未找到資料庫\",\n\n\t\"Identity address\": \"身分位址\",\n\t\"Change\": \"變更\",\n\n\t\"Site control\": \"網站控制\",\n\t\"Update\": \"更新\",\n\t\"Pause\": \"暫停\",\n\t\"Resume\": \"恢復\",\n\t\"Delete\": \"刪除\",\n\t\"Are you sure?\": \"你確定？\",\n\n\t\"Site address\": \"網站位址\",\n\t\"Donate\": \"捐贈\",\n\n\t\"Missing files\": \"缺少的檔案\",\n\t\"{} try\": \"{} 嘗試\",\n\t\"{} tries\": \"{} 已嘗試\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} 更多\",\n\n\t\"This is my site\": \"這是我的網站\",\n\t\"Site title\": \"網站標題\",\n\t\"Site description\": \"網站描述\",\n\t\"Save site settings\": \"存儲網站設定\",\n\t\"Open site directory\": \"打開所在資料夾\",\n\n\t\"Content publishing\": \"內容發布\",\n\t\"Choose\": \"選擇\",\n\t\"Sign\": \"簽署\",\n\t\"Publish\": \"發布\",\n\t\"Sign and publish\": \"簽名並發布\",\n\t\"This function is disabled on this proxy\": \"此代理上禁用此功能\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 地理位置資料庫下載錯誤：{}!
請手動下載並解壓到數據目錄：
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"正在下載 GeoLite2 地理位置資料庫 （僅一次，約 20MB ）...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 地理位置資料庫已下載！\",\n\n\t\"Are you sure?\": \"你確定？\",\n\t\"Site storage limit modified!\": \"網站存儲限制已變更！\",\n\t\"Database schema reloaded!\": \"資料庫架構重新加載！\",\n\t\"Database rebuilding....\": \"資料庫重建中...\",\n\t\"Database rebuilt!\": \"資料庫已重建！\",\n\t\"Site updated!\": \"網站已更新！\",\n\t\"Delete this site\": \"刪除此網站\",\n\t\"File write error: \": \"檔案寫入錯誤：\",\n\t\"Site settings saved!\": \"網站設置已保存！\",\n\t\"Enter your private key:\": \"輸入您的私鑰：\",\n\t\" Signed!\": \" 已簽署！\",\n\t\"WebGL not supported\": \"不支援 WebGL\"\n}\n" }, { "path": "plugins/Sidebar/languages/zh.json", "content": "{\n\t\"Copy to clipboard\": \"复制到剪切板\",\n\t\"Peers\": \"节点数\",\n\t\"Connected\": \"已连接\",\n\t\"Connectable\": \"可连接\",\n\t\"Onion\": \"洋葱点\",\n\t\"Local\": \"局域网\",\n\t\"Connectable peers\": \"可连接节点\",\n\n\t\"Data transfer\": \"数据传输\",\n\t\"Received\": \"已接收\",\n\t\"Received bytes\": \"已接收字节\",\n\t\"Sent\": \"已发送\",\n\t\"Sent bytes\": \"已发送字节\",\n\n\t\"Files\": \"文件\",\n\t\"Save as .zip\": \"打包成zip文件\",\n\t\"Total\": \"总计\",\n\t\"Image\": \"图像\",\n\t\"Other\": \"其他\",\n\t\"User data\": \"用户数据\",\n\n\t\"Size limit\": \"大小限制\",\n\t\"limit used\": \"限额\",\n\t\"free space\": \"剩余空间\",\n\t\"Set\": \"设置\",\n\n\t\"Optional files\": \"可选文件\",\n\t\"Downloaded\": \"已下载\",\n\t\"Help distribute added optional files\": \"帮助分发新的可选文件\",\n\t\"Auto download big file size limit\": \"自动下载大文件大小限制\",\n\t\"Download previous files\": \"下载之前的文件\",\n\t\"Optional files download started\": \"可选文件下载启动\",\n\t\"Optional files downloaded\": \"可选文件下载完成\",\n\t\"Total size\": \"总大小\",\n\t\"Downloaded files\": \"已下载文件\",\n\n\t\"Database\": \"数据库\",\n\t\"search feeds\": \"搜索数据源\",\n\t\"{feeds} query\": \"{feeds} 请求\",\n\t\"Reload\": \"重载\",\n\t\"Rebuild\": \"重建\",\n\t\"No database found\": \"没有找到数据库\",\n\n\t\"Identity address\": \"身份地址\",\n\t\"Change\": \"更改\",\n\n\t\"Site control\": \"站点控制\",\n\t\"Update\": \"更新\",\n\t\"Pause\": \"暂停\",\n\t\"Resume\": \"恢复\",\n\t\"Delete\": \"删除\",\n\t\"Are you sure?\": \"您确定吗？\",\n\n\t\"Site address\": \"站点地址\",\n\t\"Donate\": \"捐赠\",\n\n\t\"Needs to be updated\": \"需要更新\",\n\t\"{} try\": \"{} 尝试\",\n\t\"{} tries\": \"{} 已尝试\",\n\t\"+ {num_bad_files} more\": \"+ {num_bad_files} 更多\",\n\n\t\"This is my site\": \"这是我的站点\",\n\t\"Site title\": \"站点标题\",\n\t\"Site description\": \"站点描述\",\n\t\"Save site settings\": \"保存站点设置\",\n\t\"Open site directory\": \"打开所在文件夹\",\n\n\t\"Content publishing\": \"内容发布\",\n\t\"Add saved private key\": \"添加并保存私钥\",\n\t\"Save\": \"保存\",\n\t\"Private key saved.\": \"私钥已保存\",\n\t\"Private key saved for site signing\": \"已保存用于站点签名的私钥\",\n\t\"Forgot\": \"删除私钥\",\n\t\"Saved private key removed\": \"保存的私钥已删除\",\n\t\"Choose\": \"选择\",\n\t\"Sign\": \"签名\",\n\t\"Publish\": \"发布\",\n\t\"Sign and publish\": \"签名并发布\",\n\t\"This function is disabled on this proxy\": \"此功能在代理上被禁用\",\n\t\"GeoLite2 City database download error: {}!
Please download manually and unpack to data dir:
{}\": \"GeoLite2 地理位置数据库下载错误：{}!
请手动下载并解压在数据目录：
{}\",\n\t\"Downloading GeoLite2 City database (one time only, ~20MB)...\": \"正在下载 GeoLite2 地理位置数据库 （仅需一次，约 20MB ）...\",\n\t\"GeoLite2 City database downloaded!\": \"GeoLite2 地理位置数据库已下载！\",\n\n\t\"Are you sure?\": \"您确定吗？\",\n\t\"Site storage limit modified!\": \"站点存储限制已更改！\",\n\t\"Database schema reloaded!\": \"数据库模式已重新加载！\",\n\t\"Database rebuilding....\": \"数据库重建中...\",\n\t\"Database rebuilt!\": \"数据库已重建！\",\n\t\"Site updated!\": \"站点已更新！\",\n\t\"Delete this site\": \"删除此站点\",\n\t\"Blacklist\": \"黑名单\",\n\t\"Blacklist this site\": \"拉黑此站点\",\n\t\"Reason\": \"原因\",\n\t\"Delete and Blacklist\": \"删除并拉黑\",\n\t\"File write error: \": \"文件写入错误：\",\n\t\"Site settings saved!\": \"站点设置已保存！\",\n\t\"Enter your private key:\": \"输入您的私钥：\",\n\t\" Signed!\": \" 已签名！\",\n\t\"WebGL not supported\": \"不支持 WebGL\"\n}\n" }, { "path": "plugins/Sidebar/media/Class.coffee", "content": "class Class\n\ttrace: true\n\n\tlog: (args...) ->\n\t\treturn unless @trace\n\t\treturn if typeof console is 'undefined'\n\t\targs.unshift(\"[#{@.constructor.name}]\")\n\t\tconsole.log(args...)\n\t\t@\n\t\t\n\tlogStart: (name, args...) ->\n\t\treturn unless @trace\n\t\t@logtimers or= {}\n\t\t@logtimers[name] = +(new Date)\n\t\t@log \"#{name}\", args..., \"(started)\" if args.length > 0\n\t\t@\n\t\t\n\tlogEnd: (name, args...) ->\n\t\tms = +(new Date)-@logtimers[name]\n\t\t@log \"#{name}\", args..., \"(Done in #{ms}ms)\"\n\t\t@ \n\nwindow.Class = Class" }, { "path": "plugins/Sidebar/media/Console.coffee", "content": "class Console extends Class\n\tconstructor: (@sidebar) ->\n\t\t@tag = null\n\t\t@opened = false\n\t\t@filter = null\n\t\t@tab_types = [\n\t\t\t{title: \"All\", filter: \"\"},\n\t\t\t{title: \"Info\", filter: \"INFO\"},\n\t\t\t{title: \"Warning\", filter: \"WARNING\"},\n\t\t\t{title: \"Error\", filter: \"ERROR\"}\n\t\t]\n\t\t@read_size = 32 * 1024\n\t\t@tab_active = \"\"\n\t\t#@filter = @sidebar.wrapper.site_info.address_short\n\t\thandleMessageWebsocket_original = @sidebar.wrapper.handleMessageWebsocket\n\t\t@sidebar.wrapper.handleMessageWebsocket = (message) =>\n\t\t\tif message.cmd == \"logLineAdd\" and message.params.stream_id == @stream_id\n\t\t\t\t@addLines(message.params.lines)\n\t\t\telse\n\t\t\t\thandleMessageWebsocket_original(message)\n\n\t\t$(window).on \"hashchange\", =>\n\t\t\tif window.top.location.hash.startsWith(\"#ZeroNet:Console\")\n\t\t\t\t@open()\n\n\t\tif window.top.location.hash.startsWith(\"#ZeroNet:Console\")\n\t\t\tsetTimeout (=> @open()), 10\n\n\tcreateHtmltag: ->\n\t\tif not @container\n\t\t\t@container = $(\"\"\"\n\t\t\t
\n\t\t\t\t
\n\t\t\t\n\t\t\t\"\"\")\n\t\t\t@text = @container.find(\".console-text\")\n\t\t\t@text_elem = @text[0]\n\t\t\t@tabs = @container.find(\".console-tabs\")\n\n\t\t\t@text.on \"mousewheel\", (e) => # Stop animation on manual scrolling\n\t\t\t\tif e.originalEvent.deltaY < 0\n\t\t\t\t\t@text.stop()\n\t\t\t\tRateLimit 300, @checkTextIsBottom\n\n\t\t\t@text.is_bottom = true\n\n\t\t\t@container.appendTo(document.body)\n\t\t\t@tag = @container.find(\".console\")\n\t\t\tfor tab_type in @tab_types\n\t\t\t\ttab = $(\"\", {href: \"#\", \"data-filter\": tab_type.filter, \"data-title\": tab_type.title}).text(tab_type.title)\n\t\t\t\tif tab_type.filter == @tab_active\n\t\t\t\t\ttab.addClass(\"active\")\n\t\t\t\ttab.on(\"click\", @handleTabClick)\n\t\t\t\tif window.top.location.hash.endsWith(tab_type.title)\n\t\t\t\t\t@log \"Triggering click on\", tab\n\t\t\t\t\ttab.trigger(\"click\")\n\t\t\t\t@tabs.append(tab)\n\n\t\t\t@container.on \"mousedown touchend touchcancel\", (e) =>\n\t\t\t\tif e.target != e.currentTarget\n\t\t\t\t\treturn true\n\t\t\t\t@log \"closing\"\n\t\t\t\tif $(document.body).hasClass(\"body-console\")\n\t\t\t\t\t@close()\n\t\t\t\t\treturn true\n\n\t\t\t@loadConsoleText()\n\n\tcheckTextIsBottom: =>\n\t\t@text.is_bottom = Math.round(@text_elem.scrollTop + @text_elem.clientHeight) >= @text_elem.scrollHeight - 15\n\n\ttoColor: (text, saturation=60, lightness=70) ->\n\t\thash = 0\n\t\tfor i in [0..text.length-1]\n\t\t\thash += text.charCodeAt(i)*i\n\t\t\thash = hash % 1777\n\t\treturn \"hsl(\" + (hash % 360) + \",#{saturation}%,#{lightness}%)\";\n\n\tformatLine: (line) =>\n\t\tmatch = line.match(/(\\[.*?\\])[ ]+(.*?)[ ]+(.*?)[ ]+(.*)/)\n\t\tif not match\n\t\t\treturn line.replace(/\\/g, \">\")\n\n\t\t[line, added, level, module, text] = line.match(/(\\[.*?\\])[ ]+(.*?)[ ]+(.*?)[ ]+(.*)/)\n\t\tadded = \"#{added}\"\n\t\tlevel = \"#{level}\"\n\t\tmodule = \"#{module}\"\n\n\t\ttext = text.replace(/(Site:[A-Za-z0-9\\.]+)/g, \"$1\")\n\t\ttext = text.replace(/\\/g, \">\")\n\t\t#text = text.replace(/( [0-9\\.]+(|s|ms))/g, \"$1\")\n\t\treturn \"#{added} #{level} #{module} #{text}\"\n\n\n\taddLines: (lines, animate=true) =>\n\t\thtml_lines = []\n\t\t@logStart \"formatting\"\n\t\tfor line in lines\n\t\t\thtml_lines.push @formatLine(line)\n\t\t@logEnd \"formatting\"\n\t\t@logStart \"adding\"\n\t\t@text.append(html_lines.join(\"\n\t\t\t\t\t
\n\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t
\n\t\t\t\t\tLoading...
\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\") + \"
\")\n\t\t@logEnd \"adding\"\n\t\tif @text.is_bottom and animate\n\t\t\t@text.stop().animate({scrollTop: @text_elem.scrollHeight - @text_elem.clientHeight + 1}, 600, 'easeInOutCubic')\n\n\n\tloadConsoleText: =>\n\t\t@sidebar.wrapper.ws.cmd \"consoleLogRead\", {filter: @filter, read_size: @read_size}, (res) =>\n\t\t\t@text.html(\"\")\n\t\t\tpos_diff = res[\"pos_end\"] - res[\"pos_start\"]\n\t\t\tsize_read = Math.round(pos_diff/1024)\n\t\t\tsize_total = Math.round(res['pos_end']/1024)\n\t\t\t@text.append(\"
\")\n\t\t\t@text.append(\"Displaying #{res.lines.length} of #{res.num_found} lines found in the last #{size_read}kB of the log file. (#{size_total}kB total)
\")\n\t\t\t@addLines res.lines, false\n\t\t\t@text_elem.scrollTop = @text_elem.scrollHeight\n\t\tif @stream_id\n\t\t\t@sidebar.wrapper.ws.cmd \"consoleLogStreamRemove\", {stream_id: @stream_id}\n\t\t@sidebar.wrapper.ws.cmd \"consoleLogStream\", {filter: @filter}, (res) =>\n\t\t\t@stream_id = res.stream_id\n\n\tclose: =>\n\t\twindow.top.location.hash = \"\"\n\t\t@sidebar.move_lock = \"y\"\n\t\t@sidebar.startDrag()\n\t\t@sidebar.stopDrag()\n\n\topen: =>\n\t\t@sidebar.startDrag()\n\t\t@sidebar.moved(\"y\")\n\t\t@sidebar.fixbutton_targety = @sidebar.page_height - @sidebar.fixbutton_inity - 50\n\t\t@sidebar.stopDrag()\n\n\tonOpened: =>\n\t\t@sidebar.onClosed()\n\t\t@log \"onOpened\"\n\n\tonClosed: =>\n\t\t$(document.body).removeClass(\"body-console\")\n\t\tif @stream_id\n\t\t\t@sidebar.wrapper.ws.cmd \"consoleLogStreamRemove\", {stream_id: @stream_id}\n\n\tcleanup: =>\n\t\tif @container\n\t\t\t@container.remove()\n\t\t\t@container = null\n\n\tstopDragY: =>\n\t\t# Animate sidebar and iframe\n\t\tif @sidebar.fixbutton_targety == @sidebar.fixbutton_inity\n\t\t\t# Closed\n\t\t\ttargety = 0\n\t\t\t@opened = false\n\t\telse\n\t\t\t# Opened\n\t\t\ttargety = @sidebar.fixbutton_targety - @sidebar.fixbutton_inity\n\t\t\t@onOpened()\n\t\t\t@opened = true\n\n\t\t# Revent sidebar transitions\n\t\tif @tag\n\t\t\t@tag.css(\"transition\", \"0.5s ease-out\")\n\t\t\t@tag.css(\"transform\", \"translateY(#{targety}px)\").one transitionEnd, =>\n\t\t\t\t@tag.css(\"transition\", \"\")\n\t\t\t\tif not @opened\n\t\t\t\t\t@cleanup()\n\t\t# Revert body transformations\n\t\t@log \"stopDragY\", \"opened:\", @opened, targety\n\t\tif not @opened\n\t\t\t@onClosed()\n\n\tchangeFilter: (filter) =>\n\t\t@filter = filter\n\t\tif @filter == \"\"\n\t\t\t@read_size = 32 * 1024\n\t\telse\n\t\t\t@read_size = 5 * 1024 * 1024\n\t\t@loadConsoleText()\n\n\thandleTabClick: (e) =>\n\t\telem = $(e.currentTarget)\n\t\t@tab_active = elem.data(\"filter\")\n\t\t$(\"a\", @tabs).removeClass(\"active\")\n\t\telem.addClass(\"active\")\n\t\t@changeFilter(@tab_active)\n\t\twindow.top.location.hash = \"#ZeroNet:Console:\" + elem.data(\"title\")\n\t\treturn false\n\nwindow.Console = Console\n" }, { "path": "plugins/Sidebar/media/Console.css", "content": ".console-container { width: 100%; z-index: 998; position: absolute; top: -100vh; padding-bottom: 100%; }\n.console-container .console { background-color: #212121; height: 100vh; transform: translateY(0px); padding-top: 80px; box-sizing: border-box; }\n\n.console-top { color: white; font-family: Consolas, monospace; font-size: 11px; line-height: 20px; height: 100%; box-sizing: border-box; letter-spacing: 0.5px;}\n.console-text { overflow-y: scroll; height: calc(100% - 10px); color: #DDD; padding: 5px; margin-top: -36px; overflow-wrap: break-word; }\n.console-tabs {\n background-color: #41193fad; position: relative; margin-right: 17px; /*backdrop-filter: blur(2px);*/\n box-shadow: -30px 0px 45px #7d2463; background: linear-gradient(-75deg, #591a48ed, #70305e66); border-bottom: 1px solid #792e6473;\n}\n.console-tabs a {\n margin-right: 5px; padding: 5px 15px; text-decoration: none; color: #AAA;\n font-size: 11px; font-family: \"Consolas\"; text-transform: uppercase; border: 1px solid #666;\n border-bottom: 0px; display: inline-block; margin: 5px; margin-bottom: 0px; background-color: rgba(0,0,0,0.5);\n}\n.console-tabs a:hover { color: #FFF }\n.console-tabs a.active { background-color: #46223c; color: #FFF }\n.console-middle {height: 0px; top: 50%; position: absolute; width: 100%; left: 50%; display: none; }\n\n.console .mynode {\n\tborder: 0.5px solid #aaa; width: 50px; height: 50px; transform: rotateZ(45deg); margin-top: -25px; margin-left: -25px;\n\topacity: 1; display: inline-block; background-color: #EEE; z-index: 9; position: absolute; outline: 5px solid #EEE;\n}\n.console .peers { width: 0px; height: 0px; position: absolute; left: -20px; top: -20px; text-align: center; }\n.console .peer { left: 0px; top: 0px; position: absolute; }\n.console .peer .icon { width: 20px; height: 20px; padding: 10px; display: inline-block; text-decoration: none; left: 200px; position: absolute; color: #666; }\n.console .peer .icon:before { content: \"\\25BC\"; position: absolute; margin-top: 3px; margin-left: -1px; opacity: 0; transition: all 0.3s }\n.console .peer .icon:hover:before { opacity: 1; transition: none }\n.console .peer .line {\n\twidth: 187px; border-top: 1px solid #CCC; position: absolute; top: 20px; left: 20px;\n\ttransform: rotateZ(334deg); transform-origin: bottom left;\n}\n" }, { "path": "plugins/Sidebar/media/Menu.coffee", "content": "class Menu\n\tconstructor: (@button) ->\n\t\t@elem = $(\".menu.template\").clone().removeClass(\"template\")\n\t\t@elem.appendTo(\"body\")\n\t\t@items = []\n\n\tshow: ->\n\t\tif window.visible_menu and window.visible_menu.button[0] == @button[0] # Same menu visible then hide it\n\t\t\twindow.visible_menu.hide()\n\t\t\t@hide()\n\t\telse\n\t\t\tbutton_pos = @button.offset()\n\t\t\tleft = button_pos.left\n\t\t\t@elem.css({\"top\": button_pos.top+@button.outerHeight(), \"left\": left})\n\t\t\t@button.addClass(\"menu-active\")\n\t\t\t@elem.addClass(\"visible\")\n\t\t\tif @elem.position().left + @elem.width() + 20 > window.innerWidth\n\t\t\t\t@elem.css(\"left\", window.innerWidth - @elem.width() - 20)\n\t\t\tif window.visible_menu then window.visible_menu.hide()\n\t\t\twindow.visible_menu = @\n\n\n\thide: ->\n\t\t@elem.removeClass(\"visible\")\n\t\t@button.removeClass(\"menu-active\")\n\t\twindow.visible_menu = null\n\n\n\taddItem: (title, cb) ->\n\t\titem = $(\".menu-item.template\", @elem).clone().removeClass(\"template\")\n\t\titem.html(title)\n\t\titem.on \"click\", =>\n\t\t\tif not cb(item)\n\t\t\t\t@hide()\n\t\t\treturn false\n\t\titem.appendTo(@elem)\n\t\t@items.push item\n\t\treturn item\n\n\n\tlog: (args...) ->\n\t\tconsole.log \"[Menu]\", args...\n\nwindow.Menu = Menu\n\n# Hide menu on outside click\n$(\"body\").on \"click\", (e) ->\n\tif window.visible_menu and e.target != window.visible_menu.button[0] and $(e.target).parent()[0] != window.visible_menu.elem[0]\n\t\twindow.visible_menu.hide()\n" }, { "path": "plugins/Sidebar/media/Menu.css", "content": ".menu {\n\tbackground-color: white; padding: 10px 0px; position: absolute; top: 0px; left: 0px; max-height: 0px; overflow: hidden; transform: translate(0px, -30px); pointer-events: none;\n\tbox-shadow: 0px 2px 8px rgba(0,0,0,0.3); border-radius: 2px; opacity: 0; transition: opacity 0.2s ease-out, transform 1s ease-out, max-height 0.2s ease-in-out;\n}\n.menu.visible { opacity: 1; max-height: 350px; transform: translate(0px, 0px); transition: opacity 0.1s ease-out, transform 0.3s ease-out, max-height 0.3s ease-in-out; pointer-events: all }\n\n.menu-item { display: block; text-decoration: none; color: black; padding: 6px 24px; transition: all 0.2s; border-bottom: none; font-weight: normal; padding-left: 30px; }\n.menu-item-separator { margin-top: 5px; border-top: 1px solid #eee }\n\n.menu-item:hover { background-color: #F6F6F6; transition: none; color: inherit; border: none }\n.menu-item:active, .menu-item:focus { background-color: #AF3BFF; color: white; transition: none }\n.menu-item.selected:before {\n\tcontent: \"L\"; display: inline-block; transform: rotateZ(45deg) scaleX(-1);\n\tfont-weight: bold; position: absolute; margin-left: -17px; font-size: 12px; margin-top: 2px;\n}\n\n@media only screen and (max-width: 800px) {\n.menu, .menu.visible { position: absolute; left: unset !important; right: 20px; }\n}" }, { "path": "plugins/Sidebar/media/Prototypes.coffee", "content": "String::startsWith = (s) -> @[...s.length] is s\nString::endsWith = (s) -> s is '' or @[-s.length..] is s\nString::capitalize = -> if @.length then @[0].toUpperCase() + @.slice(1) else \"\"\nString::repeat = (count) -> new Array( count + 1 ).join(@)\n\nwindow.isEmpty = (obj) ->\n\tfor key of obj\n\t\treturn false\n\treturn true\n" }, { "path": "plugins/Sidebar/media/RateLimit.coffee", "content": "limits = {}\ncall_after_interval = {}\nwindow.RateLimit = (interval, fn) ->\n\tif not limits[fn]\n\t\tcall_after_interval[fn] = false\n\t\tfn() # First call is not delayed\n\t\tlimits[fn] = setTimeout (->\n\t\t\tif call_after_interval[fn]\n\t\t\t\tfn()\n\t\t\tdelete limits[fn]\n\t\t\tdelete call_after_interval[fn]\n\t\t), interval\n\telse # Called within iterval, delay the call\n\t\tcall_after_interval[fn] = true\n" }, { "path": "plugins/Sidebar/media/Scrollable.js", "content": "/* via http://jsfiddle.net/elGrecode/00dgurnn/ */\n\nwindow.initScrollable = function () {\n\n var scrollContainer = document.querySelector('.scrollable'),\n scrollContentWrapper = document.querySelector('.scrollable .content-wrapper'),\n scrollContent = document.querySelector('.scrollable .content'),\n contentPosition = 0,\n scrollerBeingDragged = false,\n scroller,\n topPosition,\n scrollerHeight;\n\n function calculateScrollerHeight() {\n // *Calculation of how tall scroller should be\n var visibleRatio = scrollContainer.offsetHeight / scrollContentWrapper.scrollHeight;\n if (visibleRatio == 1)\n scroller.style.display = \"none\";\n else\n scroller.style.display = \"block\";\n return visibleRatio * scrollContainer.offsetHeight;\n }\n\n function moveScroller(evt) {\n // Move Scroll bar to top offset\n var scrollPercentage = evt.target.scrollTop / scrollContentWrapper.scrollHeight;\n topPosition = scrollPercentage * (scrollContainer.offsetHeight - 5); // 5px arbitrary offset so scroll bar doesn't move too far beyond content wrapper bounding box\n scroller.style.top = topPosition + 'px';\n }\n\n function startDrag(evt) {\n normalizedPosition = evt.pageY;\n contentPosition = scrollContentWrapper.scrollTop;\n scrollerBeingDragged = true;\n window.addEventListener('mousemove', scrollBarScroll);\n return false;\n }\n\n function stopDrag(evt) {\n scrollerBeingDragged = false;\n window.removeEventListener('mousemove', scrollBarScroll);\n }\n\n function scrollBarScroll(evt) {\n if (scrollerBeingDragged === true) {\n evt.preventDefault();\n var mouseDifferential = evt.pageY - normalizedPosition;\n var scrollEquivalent = mouseDifferential * (scrollContentWrapper.scrollHeight / scrollContainer.offsetHeight);\n scrollContentWrapper.scrollTop = contentPosition + scrollEquivalent;\n }\n }\n\n function updateHeight() {\n scrollerHeight = calculateScrollerHeight() - 10;\n scroller.style.height = scrollerHeight + 'px';\n }\n\n function createScroller() {\n // *Creates scroller element and appends to '.scrollable' div\n // create scroller element\n scroller = document.createElement(\"div\");\n scroller.className = 'scroller';\n\n // determine how big scroller should be based on content\n scrollerHeight = calculateScrollerHeight() - 10;\n\n if (scrollerHeight / scrollContainer.offsetHeight < 1) {\n // *If there is a need to have scroll bar based on content size\n scroller.style.height = scrollerHeight + 'px';\n\n // append scroller to scrollContainer div\n scrollContainer.appendChild(scroller);\n\n // show scroll path divot\n scrollContainer.className += ' showScroll';\n\n // attach related draggable listeners\n scroller.addEventListener('mousedown', startDrag);\n window.addEventListener('mouseup', stopDrag);\n }\n\n }\n\n createScroller();\n\n\n // *** Listeners ***\n scrollContentWrapper.addEventListener('scroll', moveScroller);\n\n return updateHeight;\n};" }, { "path": "plugins/Sidebar/media/Scrollbable.css", "content": ".scrollable {\n overflow: hidden;\n}\n\n.scrollable.showScroll::after {\n position: absolute;\n content: '';\n top: 5%;\n right: 7px;\n height: 90%;\n width: 3px;\n background: rgba(224, 224, 255, .3);\n}\n\n.scrollable .content-wrapper {\n width: 100%;\n height: 100%;\n padding-right: 50%;\n overflow-y: scroll;\n}\n.scroller {\n margin-top: 5px;\n z-index: 5;\n cursor: pointer;\n position: absolute;\n width: 7px;\n border-radius: 5px;\n background: #3A3A3A;\n top: 0px;\n left: 395px;\n -webkit-transition: top .08s;\n -moz-transition: top .08s;\n -ms-transition: top .08s;\n -o-transition: top .08s;\n transition: top .08s;\n}\n.scroller {\n -webkit-touch-callout: none;\n -webkit-user-select: none;\n -khtml-user-select: none;\n -moz-user-select: none;\n -ms-user-select: none;\n user-select: none;\n}\n" }, { "path": "plugins/Sidebar/media/Sidebar.coffee", "content": "class Sidebar extends Class\n\tconstructor: (@wrapper) ->\n\t\t@tag = null\n\t\t@container = null\n\t\t@opened = false\n\t\t@width = 410\n\t\t@console = new Console(@)\n\t\t@fixbutton = $(\".fixbutton\")\n\t\t@fixbutton_addx = 0\n\t\t@fixbutton_addy = 0\n\t\t@fixbutton_initx = 0\n\t\t@fixbutton_inity = 15\n\t\t@fixbutton_targetx = 0\n\t\t@move_lock = null\n\t\t@page_width = $(window).width()\n\t\t@page_height = $(window).height()\n\t\t@frame = $(\"#inner-iframe\")\n\t\t@initFixbutton()\n\t\t@dragStarted = 0\n\t\t@globe = null\n\t\t@preload_html = null\n\n\t\t@original_set_site_info = @wrapper.setSiteInfo # We going to override this, save the original\n\n\t\t# Start in opened state for debugging\n\t\tif window.top.location.hash == \"#ZeroNet:OpenSidebar\"\n\t\t\t@startDrag()\n\t\t\t@moved(\"x\")\n\t\t\t@fixbutton_targetx = @fixbutton_initx - @width\n\t\t\t@stopDrag()\n\n\n\tinitFixbutton: ->\n\n\t\t# Detect dragging\n\t\t@fixbutton.on \"mousedown touchstart\", (e) =>\n\t\t\tif e.button > 0 # Right or middle click\n\t\t\t\treturn\n\t\t\te.preventDefault()\n\n\t\t\t# Disable previous listeners\n\t\t\t@fixbutton.off \"click touchend touchcancel\"\n\n\t\t\t# Make sure its not a click\n\t\t\t@dragStarted = (+ new Date)\n\n\t\t\t# Fullscreen drag bg to capture mouse events over iframe\n\t\t\t$(\".drag-bg\").remove()\n\t\t\t$(\"\").appendTo(document.body)\n\n\t\t\t$(\"body\").one \"mousemove touchmove\", (e) =>\n\t\t\t\tmousex = e.pageX\n\t\t\t\tmousey = e.pageY\n\t\t\t\tif not mousex\n\t\t\t\t\tmousex = e.originalEvent.touches[0].pageX\n\t\t\t\t\tmousey = e.originalEvent.touches[0].pageY\n\n\t\t\t\t@fixbutton_addx = @fixbutton.offset().left - mousex\n\t\t\t\t@fixbutton_addy = @fixbutton.offset().top - mousey\n\t\t\t\t@startDrag()\n\t\t@fixbutton.parent().on \"click touchend touchcancel\", (e) =>\n\t\t\tif (+ new Date) - @dragStarted < 100\n\t\t\t\twindow.top.location = @fixbutton.find(\".fixbutton-bg\").attr(\"href\")\n\t\t\t@stopDrag()\n\t\t@resized()\n\t\t$(window).on \"resize\", @resized\n\n\tresized: =>\n\t\t@page_width = $(window).width()\n\t\t@page_height = $(window).height()\n\t\t@fixbutton_initx = @page_width - 75 # Initial x position\n\t\tif @opened\n\t\t\t@fixbutton.css\n\t\t\t\tleft: @fixbutton_initx - @width\n\t\telse\n\t\t\t@fixbutton.css\n\t\t\t\tleft: @fixbutton_initx\n\n\t# Start dragging the fixbutton\n\tstartDrag: ->\n\t\t#@move_lock = \"x\" # Temporary until internals not finished\n\t\t@log \"startDrag\", @fixbutton_initx, @fixbutton_inity\n\t\t@fixbutton_targetx = @fixbutton_initx # Fallback x position\n\t\t@fixbutton_targety = @fixbutton_inity # Fallback y position\n\n\t\t@fixbutton.addClass(\"dragging\")\n\n\t\t# IE position wrap fix\n\t\tif navigator.userAgent.indexOf('MSIE') != -1 or navigator.appVersion.indexOf('Trident/') > 0\n\t\t\t@fixbutton.css(\"pointer-events\", \"none\")\n\n\t\t# Don't go to homepage\n\t\t@fixbutton.one \"click\", (e) =>\n\t\t\t@stopDrag()\n\t\t\t@fixbutton.removeClass(\"dragging\")\n\t\t\tmoved_x = Math.abs(@fixbutton.offset().left - @fixbutton_initx)\n\t\t\tmoved_y = Math.abs(@fixbutton.offset().top - @fixbutton_inity)\n\t\t\tif moved_x > 5 or moved_y > 10\n\t\t\t\t# If moved more than some pixel the button then don't go to homepage\n\t\t\t\te.preventDefault()\n\n\t\t# Animate drag\n\t\t@fixbutton.parents().on \"mousemove touchmove\", @animDrag\n\t\t@fixbutton.parents().on \"mousemove touchmove\" ,@waitMove\n\n\t\t# Stop dragging listener\n\t\t@fixbutton.parents().one \"mouseup touchend touchcancel\", (e) =>\n\t\t\te.preventDefault()\n\t\t\t@stopDrag()\n\n\n\t# Wait for moving the fixbutton\n\twaitMove: (e) =>\n\t\tdocument.body.style.perspective = \"1000px\"\n\t\tdocument.body.style.height = \"100%\"\n\t\tdocument.body.style.willChange = \"perspective\"\n\t\tdocument.documentElement.style.height = \"100%\"\n\t\t#$(document.body).css(\"backface-visibility\", \"hidden\").css(\"perspective\", \"1000px\").css(\"height\", \"900px\")\n\t\t# $(\"iframe\").css(\"backface-visibility\", \"hidden\")\n\n\t\tmoved_x = Math.abs(parseInt(@fixbutton[0].style.left) - @fixbutton_targetx)\n\t\tmoved_y = Math.abs(parseInt(@fixbutton[0].style.top) - @fixbutton_targety)\n\t\tif moved_x > 5 and (+ new Date) - @dragStarted + moved_x > 50\n\t\t\t@moved(\"x\")\n\t\t\t@fixbutton.stop().animate {\"top\": @fixbutton_inity}, 1000\n\t\t\t@fixbutton.parents().off \"mousemove touchmove\" ,@waitMove\n\n\t\telse if moved_y > 5 and (+ new Date) - @dragStarted + moved_y > 50\n\t\t\t@moved(\"y\")\n\t\t\t@fixbutton.parents().off \"mousemove touchmove\" ,@waitMove\n\n\tmoved: (direction) ->\n\t\t@log \"Moved\", direction\n\t\t@move_lock = direction\n\t\tif direction == \"y\"\n\t\t\t$(document.body).addClass(\"body-console\")\n\t\t\treturn @console.createHtmltag()\n\t\t@createHtmltag()\n\t\t$(document.body).addClass(\"body-sidebar\")\n\t\t@container.on \"mousedown touchend touchcancel\", (e) =>\n\t\t\tif e.target != e.currentTarget\n\t\t\t\treturn true\n\t\t\t@log \"closing\"\n\t\t\tif $(document.body).hasClass(\"body-sidebar\")\n\t\t\t\t@close()\n\t\t\t\treturn true\n\n\t\t$(window).off \"resize\"\n\t\t$(window).on \"resize\", =>\n\t\t\t$(document.body).css \"height\", $(window).height()\n\t\t\t@scrollable()\n\t\t\t@resized()\n\n\t\t# Override setsiteinfo to catch changes\n\t\t@wrapper.setSiteInfo = (site_info) =>\n\t\t\t@setSiteInfo(site_info)\n\t\t\t@original_set_site_info.apply(@wrapper, arguments)\n\n\t\t# Preload world.jpg\n\t\timg = new Image();\n\t\timg.src = \"/uimedia/globe/world.jpg\";\n\n\tsetSiteInfo: (site_info) ->\n\t\tRateLimit 1500, =>\n\t\t\t@updateHtmlTag()\n\t\tRateLimit 30000, =>\n\t\t\t@displayGlobe()\n\n\t# Create the sidebar html tag\n\tcreateHtmltag: ->\n\t\t@when_loaded = $.Deferred()\n\t\tif not @container\n\t\t\t@container = $(\"\"\"\n\t\t\t\n\t\t\t\"\"\")\n\t\t\t@container.appendTo(document.body)\n\t\t\t@tag = @container.find(\".sidebar\")\n\t\t\t@updateHtmlTag()\n\t\t\t@scrollable = window.initScrollable()\n\n\n\tupdateHtmlTag: ->\n\t\tif @preload_html\n\t\t\t@setHtmlTag(@preload_html)\n\t\t\t@preload_html = null\n\t\telse\n\t\t\t@wrapper.ws.cmd \"sidebarGetHtmlTag\", {}, @setHtmlTag\n\n\tsetHtmlTag: (res) =>\n\t\tif @tag.find(\".content\").children().length == 0 # First update\n\t\t\t@log \"Creating content\"\n\t\t\t@container.addClass(\"loaded\")\n\t\t\tmorphdom(@tag.find(\".content\")[0], '
'+res+'
')\n\t\t\t# @scrollable()\n\t\t\t@when_loaded.resolve()\n\n\t\telse # Not first update, patch the html to keep unchanged dom elements\n\t\t\tmorphdom @tag.find(\".content\")[0], ''+res+'
', {\n\t\t\t\tonBeforeMorphEl: (from_el, to_el) -> # Ignore globe loaded state\n\t\t\t\t\tif from_el.className == \"globe\" or from_el.className.indexOf(\"noupdate\") >= 0\n\t\t\t\t\t\treturn false\n\t\t\t\t\telse\n\t\t\t\t\t\treturn true\n\t\t\t\t}\n\n\t\t# Save and forget privatekey for site signing\n\t\t@tag.find(\"#privatekey-add\").off(\"click, touchend\").on \"click touchend\", (e) =>\n\t\t\t@wrapper.displayPrompt \"Enter your private key:\", \"password\", \"Save\", \"\", (privatekey) =>\n\t\t\t\t@wrapper.ws.cmd \"userSetSitePrivatekey\", [privatekey], (res) =>\n\t\t\t\t\t@wrapper.notifications.add \"privatekey\", \"done\", \"Private key saved for site signing\", 5000\n\t\t\treturn false\n\n\t\t@tag.find(\"#privatekey-forget\").off(\"click, touchend\").on \"click touchend\", (e) =>\n\t\t\t@wrapper.displayConfirm \"Remove saved private key for this site?\", \"Forget\", (res) =>\n\t\t\t\tif not res\n\t\t\t\t\treturn false\n\t\t\t\t@wrapper.ws.cmd \"userSetSitePrivatekey\", [\"\"], (res) =>\n\t\t\t\t\t@wrapper.notifications.add \"privatekey\", \"done\", \"Saved private key removed\", 5000\n\t\t\treturn false\n\n\t\t# Use requested address for browse files urls\n\t\t@tag.find(\"#browse-files\").attr(\"href\", document.location.pathname.replace(/(\\/.*?(\\/|$)).*$/, \"/list$1\"))\n\n\n\n\tanimDrag: (e) =>\n\t\tmousex = e.pageX\n\t\tmousey = e.pageY\n\t\tif not mousex and e.originalEvent.touches\n\t\t\tmousex = e.originalEvent.touches[0].pageX\n\t\t\tmousey = e.originalEvent.touches[0].pageY\n\n\t\toverdrag = @fixbutton_initx - @width - mousex\n\t\tif overdrag > 0 # Overdragged\n\t\t\toverdrag_percent = 1 + overdrag/300\n\t\t\tmousex = (mousex + (@fixbutton_initx-@width)*overdrag_percent)/(1+overdrag_percent)\n\t\ttargetx = @fixbutton_initx - mousex - @fixbutton_addx\n\t\ttargety = @fixbutton_inity - mousey - @fixbutton_addy\n\n\t\tif @move_lock == \"x\"\n\t\t\ttargety = @fixbutton_inity\n\t\telse if @move_lock == \"y\"\n\t\t\ttargetx = @fixbutton_initx\n\n\t\tif not @move_lock or @move_lock == \"x\"\n\t\t\t@fixbutton[0].style.left = (mousex + @fixbutton_addx) + \"px\"\n\t\t\tif @tag\n\t\t\t\t@tag[0].style.transform = \"translateX(#{0 - targetx}px)\"\n\n\t\tif not @move_lock or @move_lock == \"y\"\n\t\t\t@fixbutton[0].style.top = (mousey + @fixbutton_addy) + \"px\"\n\t\t\tif @console.tag\n\t\t\t\t@console.tag[0].style.transform = \"translateY(#{0 - targety}px)\"\n\n\t\t#if @move_lock == \"x\"\n\t\t\t# @fixbutton[0].style.left = \"#{@fixbutton_targetx} px\"\n\t\t\t#@fixbutton[0].style.top = \"#{@fixbutton_inity}px\"\n\t\t#if @move_lock == \"y\"\n\t\t#\t@fixbutton[0].style.top = \"#{@fixbutton_targety} px\"\n\n\t\t# Check if opened\n\t\tif (not @opened and targetx > @width/3) or (@opened and targetx > @width*0.9)\n\t\t\t@fixbutton_targetx = @fixbutton_initx - @width # Make it opened\n\t\telse\n\t\t\t@fixbutton_targetx = @fixbutton_initx\n\n\t\tif (not @console.opened and 0 - targety > @page_height/10) or (@console.opened and 0 - targety > @page_height*0.8)\n\t\t\t@fixbutton_targety = @page_height - @fixbutton_inity - 50\n\t\telse\n\t\t\t@fixbutton_targety = @fixbutton_inity\n\n\n\t# Stop dragging the fixbutton\n\tstopDrag: ->\n\t\t@fixbutton.parents().off \"mousemove touchmove\"\n\t\t@fixbutton.off \"mousemove touchmove\"\n\t\t@fixbutton.css(\"pointer-events\", \"\")\n\t\t$(\".drag-bg\").remove()\n\t\tif not @fixbutton.hasClass(\"dragging\")\n\t\t\treturn\n\t\t@fixbutton.removeClass(\"dragging\")\n\n\t\t# Move back to initial position\n\t\tif @fixbutton_targetx != @fixbutton.offset().left or @fixbutton_targety != @fixbutton.offset().top\n\t\t\t# Animate fixbutton\n\t\t\tif @move_lock == \"y\"\n\t\t\t\ttop = @fixbutton_targety\n\t\t\t\tleft = @fixbutton_initx\n\t\t\tif @move_lock == \"x\"\n\t\t\t\ttop = @fixbutton_inity\n\t\t\t\tleft = @fixbutton_targetx\n\t\t\t@fixbutton.stop().animate {\"left\": left, \"top\": top}, 500, \"easeOutBack\", =>\n\t\t\t\t# Switch back to auto align\n\t\t\t\tif @fixbutton_targetx == @fixbutton_initx # Closed\n\t\t\t\t\t@fixbutton.css(\"left\", \"auto\")\n\t\t\t\telse # Opened\n\t\t\t\t\t@fixbutton.css(\"left\", left)\n\n\t\t\t\t$(\".fixbutton-bg\").trigger \"mouseout\" # Switch fixbutton back to normal status\n\n\t\t\t@stopDragX()\n\t\t\t@console.stopDragY()\n\t\t@move_lock = null\n\n\tstopDragX: ->\n\t\t# Animate sidebar and iframe\n\t\tif @fixbutton_targetx == @fixbutton_initx or @move_lock == \"y\"\n\t\t\t# Closed\n\t\t\ttargetx = 0\n\t\t\t@opened = false\n\t\telse\n\t\t\t# Opened\n\t\t\ttargetx = @width\n\t\t\tif @opened\n\t\t\t\t@onOpened()\n\t\t\telse\n\t\t\t\t@when_loaded.done =>\n\t\t\t\t\t@onOpened()\n\t\t\t@opened = true\n\n\t\t# Revent sidebar transitions\n\t\tif @tag\n\t\t\t@tag.css(\"transition\", \"0.4s ease-out\")\n\t\t\t@tag.css(\"transform\", \"translateX(-#{targetx}px)\").one transitionEnd, =>\n\t\t\t\t@tag.css(\"transition\", \"\")\n\t\t\t\tif not @opened\n\t\t\t\t\t@container.remove()\n\t\t\t\t\t@container = null\n\t\t\t\t\tif @tag\n\t\t\t\t\t\t@tag.remove()\n\t\t\t\t\t\t@tag = null\n\n\t\t# Revert body transformations\n\t\t@log \"stopdrag\", \"opened:\", @opened\n\t\tif not @opened\n\t\t\t@onClosed()\n\n\tsign: (inner_path, privatekey) ->\n\t\t@wrapper.displayProgress(\"sign\", \"Signing: #{inner_path}...\", 0)\n\t\t@wrapper.ws.cmd \"siteSign\", {privatekey: privatekey, inner_path: inner_path, update_changed_files: true}, (res) =>\n\t\t\tif res == \"ok\"\n\t\t\t\t@wrapper.displayProgress(\"sign\", \"#{inner_path} signed!\", 100)\n\t\t\telse\n\t\t\t\t@wrapper.displayProgress(\"sign\", \"Error signing #{inner_path}\", -1)\n\n\tpublish: (inner_path, privatekey) ->\n\t\t@wrapper.ws.cmd \"sitePublish\", {privatekey: privatekey, inner_path: inner_path, sign: true, update_changed_files: true}, (res) =>\n\t\t\tif res == \"ok\"\n\t\t\t\t@wrapper.notifications.add \"sign\", \"done\", \"#{inner_path} Signed and published!\", 5000\n\n\thandleSiteDeleteClick: ->\n\t\tif @wrapper.site_info.privatekey\n\t\t\tquestion = \"Are you sure?This site has a saved private key\"\n\t\t\toptions = [\"Forget private key and delete site\"]\n\t\telse\n\t\t\tquestion = \"Are you sure?\"\n\t\t\toptions = [\"Delete this site\", \"Blacklist\"]\n\t\t@wrapper.displayConfirm question, options, (confirmed) =>\n\t\t\tif confirmed == 1\n\t\t\t\t@tag.find(\"#button-delete\").addClass(\"loading\")\n\t\t\t\t@wrapper.ws.cmd \"siteDelete\", @wrapper.site_info.address, ->\n\t\t\t\t\tdocument.location = $(\".fixbutton-bg\").attr(\"href\")\n\t\t\telse if confirmed == 2\n\t\t\t\t@wrapper.displayPrompt \"Blacklist this site\", \"text\", \"Delete and Blacklist\", \"Reason\", (reason) =>\n\t\t\t\t\t@tag.find(\"#button-delete\").addClass(\"loading\")\n\t\t\t\t\t@wrapper.ws.cmd \"siteblockAdd\", [@wrapper.site_info.address, reason]\n\t\t\t\t\t@wrapper.ws.cmd \"siteDelete\", @wrapper.site_info.address, ->\n\t\t\t\t\t\tdocument.location = $(\".fixbutton-bg\").attr(\"href\")\n\n\tonOpened: ->\n\t\t@log \"Opened\"\n\t\t@scrollable()\n\n\t\t# Re-calculate height when site admin opened or closed\n\t\t@tag.find(\"#checkbox-owned, #checkbox-autodownloadoptional\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\tsetTimeout (=>\n\t\t\t\t@scrollable()\n\t\t\t), 300\n\n\t\t# Site limit button\n\t\t@tag.find(\"#button-sitelimit\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"siteSetLimit\", $(\"#input-sitelimit\").val(), (res) =>\n\t\t\t\tif res == \"ok\"\n\t\t\t\t\t@wrapper.notifications.add \"done-sitelimit\", \"done\", \"Site storage limit modified!\", 5000\n\t\t\t\t@updateHtmlTag()\n\t\t\treturn false\n\n\t\t# Site autodownload limit button\n\t\t@tag.find(\"#button-autodownload_bigfile_size_limit\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"siteSetAutodownloadBigfileLimit\", $(\"#input-autodownload_bigfile_size_limit\").val(), (res) =>\n\t\t\t\tif res == \"ok\"\n\t\t\t\t\t@wrapper.notifications.add \"done-bigfilelimit\", \"done\", \"Site bigfile auto download limit modified!\", 5000\n\t\t\t\t@updateHtmlTag()\n\t\t\treturn false\n\n\t\t# Site start download optional files\n\t\t@tag.find(\"#button-autodownload_previous\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"siteUpdate\", {\"address\": @wrapper.site_info.address, \"check_files\": true}, =>\n\t\t\t\t@wrapper.notifications.add \"done-download_optional\", \"done\", \"Optional files downloaded\", 5000\n\n\t\t\t@wrapper.notifications.add \"start-download_optional\", \"info\", \"Optional files download started\", 5000\n\t\t\treturn false\n\n\t\t# Database reload\n\t\t@tag.find(\"#button-dbreload\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"dbReload\", [], =>\n\t\t\t\t@wrapper.notifications.add \"done-dbreload\", \"done\", \"Database schema reloaded!\", 5000\n\t\t\t\t@updateHtmlTag()\n\t\t\treturn false\n\n\t\t# Database rebuild\n\t\t@tag.find(\"#button-dbrebuild\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.notifications.add \"done-dbrebuild\", \"info\", \"Database rebuilding....\"\n\t\t\t@wrapper.ws.cmd \"dbRebuild\", [], =>\n\t\t\t\t@wrapper.notifications.add \"done-dbrebuild\", \"done\", \"Database rebuilt!\", 5000\n\t\t\t\t@updateHtmlTag()\n\t\t\treturn false\n\n\t\t# Update site\n\t\t@tag.find(\"#button-update\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@tag.find(\"#button-update\").addClass(\"loading\")\n\t\t\t@wrapper.ws.cmd \"siteUpdate\", @wrapper.site_info.address, =>\n\t\t\t\t@wrapper.notifications.add \"done-updated\", \"done\", \"Site updated!\", 5000\n\t\t\t\t@tag.find(\"#button-update\").removeClass(\"loading\")\n\t\t\treturn false\n\n\t\t# Pause site\n\t\t@tag.find(\"#button-pause\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@tag.find(\"#button-pause\").addClass(\"hidden\")\n\t\t\t@wrapper.ws.cmd \"sitePause\", @wrapper.site_info.address\n\t\t\treturn false\n\n\t\t# Resume site\n\t\t@tag.find(\"#button-resume\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@tag.find(\"#button-resume\").addClass(\"hidden\")\n\t\t\t@wrapper.ws.cmd \"siteResume\", @wrapper.site_info.address\n\t\t\treturn false\n\n\t\t# Delete site\n\t\t@tag.find(\"#button-delete\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@handleSiteDeleteClick()\n\t\t\treturn false\n\n\t\t# Owned checkbox\n\t\t@tag.find(\"#checkbox-owned\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\towned = @tag.find(\"#checkbox-owned\").is(\":checked\")\n\t\t\t@wrapper.ws.cmd \"siteSetOwned\", [owned], (res_set_owned) =>\n\t\t\t\t@log \"Owned\", owned\n\t\t\t\tif owned\n\t\t\t\t\t@wrapper.ws.cmd \"siteRecoverPrivatekey\", [], (res_recover) =>\n\t\t\t\t\t\tif res_recover == \"ok\"\n\t\t\t\t\t\t\t@wrapper.notifications.add(\"recover\", \"done\", \"Private key recovered from master seed\", 5000)\n\t\t\t\t\t\telse\n\t\t\t\t\t\t\t@log \"Unable to recover private key: #{res_recover.error}\"\n\n\n\t\t# Owned auto download checkbox\n\t\t@tag.find(\"#checkbox-autodownloadoptional\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"siteSetAutodownloadoptional\", [@tag.find(\"#checkbox-autodownloadoptional\").is(\":checked\")]\n\n\t\t# Change identity button\n\t\t@tag.find(\"#button-identity\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"certSelect\"\n\t\t\treturn false\n\n\t\t# Save settings\n\t\t@tag.find(\"#button-settings\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"fileGet\", \"content.json\", (res) =>\n\t\t\t\tdata = JSON.parse(res)\n\t\t\t\tdata[\"title\"] = $(\"#settings-title\").val()\n\t\t\t\tdata[\"description\"] = $(\"#settings-description\").val()\n\t\t\t\tjson_raw = unescape(encodeURIComponent(JSON.stringify(data, undefined, '\\t')))\n\t\t\t\t@wrapper.ws.cmd \"fileWrite\", [\"content.json\", btoa(json_raw), true], (res) =>\n\t\t\t\t\tif res != \"ok\" # fileWrite failed\n\t\t\t\t\t\t@wrapper.notifications.add \"file-write\", \"error\", \"File write error: #{res}\"\n\t\t\t\t\telse\n\t\t\t\t\t\t@wrapper.notifications.add \"file-write\", \"done\", \"Site settings saved!\", 5000\n\t\t\t\t\t\tif @wrapper.site_info.privatekey\n\t\t\t\t\t\t\t@wrapper.ws.cmd \"siteSign\", {privatekey: \"stored\", inner_path: \"content.json\", update_changed_files: true}\n\t\t\t\t\t\t@updateHtmlTag()\n\t\t\treturn false\n\n\n\t\t# Open site directory\n\t\t@tag.find(\"#link-directory\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\t@wrapper.ws.cmd \"serverShowdirectory\", [\"site\", @wrapper.site_info.address]\n\t\t\treturn false\n\n\t\t# Copy site with peers\n\t\t@tag.find(\"#link-copypeers\").off(\"click touchend\").on \"click touchend\", (e) =>\n\t\t\tcopy_text = e.currentTarget.href\n\t\t\thandler = (e) =>\n\t\t\t\te.clipboardData.setData('text/plain', copy_text)\n\t\t\t\te.preventDefault()\n\t\t\t\t@wrapper.notifications.add \"copy\", \"done\", \"Site address with peers copied to your clipboard\", 5000\n\t\t\t\tdocument.removeEventListener('copy', handler, true)\n\n\t\t\tdocument.addEventListener('copy', handler, true)\n\t\t\tdocument.execCommand('copy')\n\t\t\treturn false\n\n\t\t# Sign and publish content.json\n\t\t$(document).on \"click touchend\", =>\n\t\t\t@tag?.find(\"#button-sign-publish-menu\").removeClass(\"visible\")\n\t\t\t@tag?.find(\".contents + .flex\").removeClass(\"sign-publish-flex\")\n\n\t\t@tag.find(\".contents-content\").off(\"click touchend\").on \"click touchend\", (e) =>\n\t\t\t$(\"#input-contents\").val(e.currentTarget.innerText);\n\t\t\treturn false;\n\n\t\tmenu = new Menu(@tag.find(\"#menu-sign-publish\"))\n\t\tmenu.elem.css(\"margin-top\", \"-130px\") # Open upwards\n\t\tmenu.addItem \"Sign\", =>\n\t\t\tinner_path = @tag.find(\"#input-contents\").val()\n\n\t\t\t@wrapper.ws.cmd \"fileRules\", {inner_path: inner_path}, (rules) =>\n\t\t\t\tif @wrapper.site_info.auth_address in rules.signers\n\t\t\t\t\t# ZeroID or other ID provider\n\t\t\t\t\t@sign(inner_path)\n\t\t\t\telse if @wrapper.site_info.privatekey\n\t\t\t\t\t# Privatekey stored in users.json\n\t\t\t\t\t@sign(inner_path, \"stored\")\n\t\t\t\telse\n\t\t\t\t\t# Ask the user for privatekey\n\t\t\t\t\t@wrapper.displayPrompt \"Enter your private key:\", \"password\", \"Sign\", \"\", (privatekey) => # Prompt the private key\n\t\t\t\t\t\t@sign(inner_path, privatekey)\n\n\t\t\t@tag.find(\".contents + .flex\").removeClass \"active\"\n\t\t\tmenu.hide()\n\n\t\tmenu.addItem \"Publish\", =>\n\t\t\tinner_path = @tag.find(\"#input-contents\").val()\n\t\t\t@wrapper.ws.cmd \"sitePublish\", {\"inner_path\": inner_path, \"sign\": false}\n\n\t\t\t@tag.find(\".contents + .flex\").removeClass \"active\"\n\t\t\tmenu.hide()\n\n\t\t@tag.find(\"#menu-sign-publish\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\tif window.visible_menu == menu\n\t\t\t\t@tag.find(\".contents + .flex\").removeClass \"active\"\n\t\t\t\tmenu.hide()\n\t\t\telse\n\t\t\t\t@tag.find(\".contents + .flex\").addClass \"active\"\n\t\t\t\t@tag.find(\".content-wrapper\").prop \"scrollTop\", 10000\n\t\t\t\tmenu.show()\n\t\t\treturn false\n\n\t\t$(\"body\").on \"click\", =>\n\t\t\tif @tag\n\t\t\t\t@tag.find(\".contents + .flex\").removeClass \"active\"\n\n\t\t@tag.find(\"#button-sign-publish\").off(\"click touchend\").on \"click touchend\", =>\n\t\t\tinner_path = @tag.find(\"#input-contents\").val()\n\n\t\t\t@wrapper.ws.cmd \"fileRules\", {inner_path: inner_path}, (rules) =>\n\t\t\t\tif @wrapper.site_info.auth_address in rules.signers\n\t\t\t\t\t# ZeroID or other ID provider\n\t\t\t\t\t@publish(inner_path, null)\n\t\t\t\telse if @wrapper.site_info.privatekey\n\t\t\t\t\t# Privatekey stored in users.json\n\t\t\t\t\t@publish(inner_path, \"stored\")\n\t\t\t\telse\n\t\t\t\t\t# Ask the user for privatekey\n\t\t\t\t\t@wrapper.displayPrompt \"Enter your private key:\", \"password\", \"Sign\", \"\", (privatekey) => # Prompt the private key\n\t\t\t\t\t\t@publish(inner_path, privatekey)\n\t\t\treturn false\n\n\t\t# Close\n\t\t@tag.find(\".close\").off(\"click touchend\").on \"click touchend\", (e) =>\n\t\t\t@close()\n\t\t\treturn false\n\n\t\t@loadGlobe()\n\n\tclose: ->\n\t\t@move_lock = \"x\"\n\t\t@startDrag()\n\t\t@stopDrag()\n\n\n\tonClosed: ->\n\t\t$(window).off \"resize\"\n\t\t$(window).on \"resize\", @resized\n\t\t$(document.body).css(\"transition\", \"0.6s ease-in-out\").removeClass(\"body-sidebar\").on transitionEnd, (e) =>\n\t\t\tif e.target == document.body and not $(document.body).hasClass(\"body-sidebar\") and not $(document.body).hasClass(\"body-console\")\n\t\t\t\t$(document.body).css(\"height\", \"auto\").css(\"perspective\", \"\").css(\"will-change\", \"\").css(\"transition\", \"\").off transitionEnd\n\t\t\t\t@unloadGlobe()\n\n\t\t# We dont need site info anymore\n\t\t@wrapper.setSiteInfo = @original_set_site_info\n\n\n\tloadGlobe: =>\n\t\tif @tag.find(\".globe\").hasClass(\"loading\")\n\t\t\tsetTimeout (=>\n\t\t\t\tif typeof(DAT) == \"undefined\" # Globe script not loaded, do it first\n\t\t\t\t\tscript_tag = $(\"\n