.*?
您好！(.*?)您有未读提醒 1 条
',
html, re.S)[0]
print(data)
if __name__ == '__main__':
username = input('请输入账户:')
pwd = input('请输入密码：')
sign = main(pwd)
print('正在登录....')
login(sign, username)
home()
================================================
FILE: 其他实战/【谷雨】数字解密/GuYu.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-09-25 Python: 3.7
import requests
import os
from fontTools.ttLib import TTFont
class Font:
"""
https://guyujiezi.com/
谷雨解字的 数字解密
现在版本的 雨谷字体加的xml 会有一个移位操作
"""
def __init__(self, uri):
self.url = uri
self.filename = uri.split('/')[-1]
self.font = None
self._list = []
def check(self):
"""检查目录
"""
if not os.path.isfile(self.filename):
resp = requests.get(self.url)
with open(self.filename, 'wb') as f:
f.write(resp.content)
# TTFont 存为 xml
self.font = TTFont(self.filename)
self.font.saveXML(self.filename.replace(self.filename.split('.')[-1], 'xml'))
def get_wo(self):
"""获取 woff
"""
self.check()
ph = self.font['cmap']
_dict = ph.tables[0].cmap
# 1. 字典取 value 列表化
# 2. str 取最后 2 位，并转为 int
# 3. 减去 17 并从新组装列表
self._list = [int(i[-2:])-17 for i in list(_dict.values())]
"""
处理移位
"""
print(list(_dict.values()))
print(self._list)
def parse(self, number):
_str = ''
for num in number:
_str += str(self._list[int(num)])
print('最终展示字', int(_str))
if __name__ == '__main__':
ft = Font("https://guyujiezi.com/fonts/2DLw9u/3iZbr8.woff")
ft.get_wo()
# 输入页面数字测试
ft.parse('947')
================================================
FILE: 其他实战/【豆瓣】自动登录/DouBan.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2020-01-08 Python: 3.7
import requests
import re
class DouBan:
def __init__(self, name, pwd):
self.name = name.strip()
self.pwd = pwd.strip()
self.session = requests.session()
self.headers = {
'Origin': 'https://accounts.douban.com',
'Host': 'accounts.douban.com',
'Referer': 'https://accounts.douban.com/passport/login_popup?login_source=anony',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
}
self.login_url = 'https://accounts.douban.com/j/mobile/login/basic'
self.index_url = "https://www.douban.com/"
self.session = requests.session()
def login(self):
data = {
'ck': '',
'name': self.name,
'password': self.pwd,
'remember': 'false',
'ticket': '',
}
self.session.post(self.login_url, data=data, headers=self.headers)
def check(self):
self.headers['Host'] = 'www.douban.com'
response = self.session.get("https://www.douban.com/", headers=self.headers)
try:
title = re.search(r'
(.*?)的帐号', response.text).group(1)
print('【登录成功】', title)
except:
print('【登录失败】')
if __name__ == '__main__':
username = input('豆瓣用户名 >>>')
password = input('密码 >>>')
db = DouBan(username, password)
db.login()
db.check()
================================================
FILE: 其他实战/【逗游】自动登录/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-06 Python: 3.7
================================================
FILE: 其他实战/【逗游】自动登录/douyou.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-08-01 Python: 3.7
import js2py
import requests
import json
class DouYou:
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',
'Referer': 'http://www.doyo.cn/passport/login'
}
def __init__(self, username, password):
self.context = js2py.EvalJs() # python中使用js
self.username = username
self.password = password
def make_password(self):
"""取加密后的字符串
"""
try:
nonce, ts = self.get_token()
with open("encryp.js", "r", encoding="utf-8") as f:
self.context.execute(f.read())
pwd_hash = self.context.get_value(self.password, nonce, ts)
return pwd_hash # 打印加密之后的密码
except:
print('获取token失败')
def get_token(self):
"""获取 token
"""
get_token_url = 'http://www.doyo.cn/User/Passport/token?username={user}&random=0.1428378278012199'.format(user=self.username)
result = json.loads(requests.get(get_token_url).text)
if result.get('result'):
nonce = result.get('nonce')
ts = result.get('ts')
return nonce, ts
else:
print('获取token失败')
exit()
def login(self):
"""登陆
"""
# decode('unicode_escape')
login_url = 'http://www.doyo.cn/passport/login'
data = {
'username': self.username,
'password': self.make_password(),
'remberme': '1',
'next': 'aHR0cCUzQSUyRiUyRnd3dy5kb3lvLmNuJTJG'
}
response = requests.post(login_url, data=data, headers=self.headers)
info = json.loads(response.text)
if info.get('result'):
print('登陆成功 | 用户等级:{level} 用户id:{uid}'.format(level=info.get('level'), uid=info.get('uid')))
else:
print('登陆失败')
if __name__ == '__main__':
user = input('输入逗游账号')
pwd = input('输入密码')
dy = DouYou(user, pwd)
dy.login()
================================================
FILE: 其他实战/【逗游】自动登录/encryp.js
================================================
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* SHA-1 implementation in JavaScript | (c) Chris Veness 2002-2010 | www.movable-type.co.uk */
/* - see http://csrc.nist.gov/groups/ST/toolkit/secure_hashing.html */
/* http://csrc.nist.gov/groups/ST/toolkit/examples.html */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
var Sha1 = {}; // Sha1 namespace
/**
* Generates SHA-1 hash of string
*
* @param {String} msg String to be hashed
* @param {Boolean} [utf8encode=true] Encode msg as UTF-8 before generating hash
* @returns {String} Hash of msg as hex character string
*/
Sha1.hash = function(msg, utf8encode) {
utf8encode = (typeof utf8encode == 'undefined') ? true : utf8encode;
// convert string to UTF-8, as SHA only deals with byte-streams
if (utf8encode) msg = Utf8.encode(msg);
// constants [§4.2.1]
var K = [0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6];
// PREPROCESSING
msg += String.fromCharCode(0x80); // add trailing '1' bit (+ 0's padding) to string [§5.1.1]
// convert string msg into 512-bit/16-integer blocks arrays of ints [§5.2.1]
var l = msg.length/4 + 2; // length (in 32-bit integers) of msg + ‘1’ + appended length
var N = Math.ceil(l/16); // number of 16-integer-blocks required to hold 'l' ints
var M = new Array(N);
for (var i=0; i
>> 32, but since JS converts
// bitwise-op args to 32 bits, we need to simulate this by arithmetic operators
M[N-1][14] = ((msg.length-1)*8) / Math.pow(2, 32); M[N-1][14] = Math.floor(M[N-1][14])
M[N-1][15] = ((msg.length-1)*8) & 0xffffffff;
// set initial hash value [§5.3.1]
var H0 = 0x67452301;
var H1 = 0xefcdab89;
var H2 = 0x98badcfe;
var H3 = 0x10325476;
var H4 = 0xc3d2e1f0;
// HASH COMPUTATION [§6.1.2]
var W = new Array(80); var a, b, c, d, e;
for (var i=0; i>>(32-n));
}
//
// hexadecimal representation of a number
// (note toString(16) is implementation-dependant, and
// in IE returns signed numbers when used on full words)
//
Sha1.toHexStr = function(n) {
var s="", v;
for (var i=7; i>=0; i--) { v = (n>>>(i*4)) & 0xf; s += v.toString(16); }
return s;
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* Utf8 class: encode / decode between multi-byte Unicode characters and UTF-8 multiple */
/* single-byte character encoding (c) Chris Veness 2002-2010 */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
var Utf8 = {}; // Utf8 namespace
/**
* Encode multi-byte Unicode string into utf-8 multiple single-byte characters
* (BMP / basic multilingual plane only)
*
* Chars in range U+0080 - U+07FF are encoded in 2 chars, U+0800 - U+FFFF in 3 chars
*
* @param {String} strUni Unicode string to be encoded as UTF-8
* @returns {String} encoded string
*/
Utf8.encode = function(strUni) {
// use regular expressions & String.replace callback function for better efficiency
// than procedural approaches
var strUtf = strUni.replace(
/[\u0080-\u07ff]/g, // U+0080 - U+07FF => 2 bytes 110yyyyy, 10zzzzzz
function(c) {
var cc = c.charCodeAt(0);
return String.fromCharCode(0xc0 | cc>>6, 0x80 | cc&0x3f); }
);
strUtf = strUtf.replace(
/[\u0800-\uffff]/g, // U+0800 - U+FFFF => 3 bytes 1110xxxx, 10yyyyyy, 10zzzzzz
function(c) {
var cc = c.charCodeAt(0);
return String.fromCharCode(0xe0 | cc>>12, 0x80 | cc>>6&0x3F, 0x80 | cc&0x3f); }
);
return strUtf;
}
/**
* Decode utf-8 encoded string back into multi-byte Unicode characters
*
* @param {String} strUtf UTF-8 string to be decoded back to Unicode
* @returns {String} decoded string
*/
Utf8.decode = function(strUtf) {
// note: decode 3-byte chars first as decoded 2-byte strings could appear to be 3-byte char!
var strUni = strUtf.replace(
/[\u00e0-\u00ef][\u0080-\u00bf][\u0080-\u00bf]/g, // 3-byte chars
function(c) { // (note parentheses for precence)
var cc = ((c.charCodeAt(0)&0x0f)<<12) | ((c.charCodeAt(1)&0x3f)<<6) | ( c.charCodeAt(2)&0x3f);
return String.fromCharCode(cc); }
);
strUni = strUni.replace(
/[\u00c0-\u00df][\u0080-\u00bf]/g, // 2-byte chars
function(c) { // (note parentheses for precence)
var cc = (c.charCodeAt(0)&0x1f)<<6 | c.charCodeAt(1)&0x3f;
return String.fromCharCode(cc); }
);
return strUni;
}
function get_value(pwd, nonce, ts) {
pwd = Sha1.hash(pwd);
pwd = Sha1.hash(nonce+ts+pwd);
return pwd
}
================================================
FILE: 其他实战/【金逸电影】自动注册/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-06 Python: 3.7
================================================
FILE: 其他实战/【金逸电影】自动注册/encryp.js
================================================
(function (global) {
"use strict";
var _Base64 = global.Base64;
var version = "2.1.9";
var buffer;
if (typeof module !== "undefined" && module.exports) {
try {
buffer = require("buffer").Buffer
} catch (err) {
}
}
var b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
var b64tab = function (bin) {
var t = {};
for (var i = 0, l = bin.length; i < l; i++) t[bin.charAt(i)] = i;
return t
}(b64chars);
var fromCharCode = String.fromCharCode;
var cb_utob = function (c) {
if (c.length < 2) {
var cc = c.charCodeAt(0);
return cc < 128 ? c : cc < 2048 ? fromCharCode(192 | cc >>> 6) + fromCharCode(128 | cc & 63) : fromCharCode(224 |
cc >>> 12 & 15) + fromCharCode(128 | cc >>> 6 & 63) + fromCharCode(128 | cc & 63)
} else {
var cc = 65536 + (c.charCodeAt(0) - 55296) * 1024 + (c.charCodeAt(1) - 56320);
return fromCharCode(240 | cc >>> 18 & 7) + fromCharCode(128 | cc >>> 12 & 63) + fromCharCode(128 | cc >>> 6 & 63) +
fromCharCode(128 | cc & 63)
}
};
var re_utob = /[\uD800-\uDBFF][\uDC00-\uDFFFF]|[^\x00-\x7F]/g;
var utob = function (u) {
return u.replace(re_utob, cb_utob)
};
var cb_encode = function (ccc) {
var padlen = [0, 2, 1][ccc.length % 3],
ord = ccc.charCodeAt(0) << 16 | (ccc.length > 1 ? ccc.charCodeAt(1) : 0) << 8 | (ccc.length > 2 ? ccc.charCodeAt(
2) : 0),
chars = [b64chars.charAt(ord >>> 18), b64chars.charAt(ord >>> 12 & 63), padlen >= 2 ? "=" : b64chars.charAt(ord >>>
6 & 63), padlen >= 1 ? "=" : b64chars.charAt(ord & 63)];
return chars.join("")
};
var btoa = global.btoa ? function (b) {
return global.btoa(b)
} : function (b) {
return b.replace(/[\s\S]{1,3}/g, cb_encode)
};
var _encode = buffer ? function (u) {
return (u.constructor === buffer.constructor ? u : new buffer(u)).toString("base64")
} : function (u) {
return btoa(utob(u))
};
var encode = function (u, urisafe) {
return !urisafe ? _encode(String(u)) : _encode(String(u)).replace(/[+\/]/g, function (m0) {
return m0 == "+" ? "-" : "_"
}).replace(/=/g, "")
};
var encodeURI = function (u) {
return encode(u, true)
};
var re_btou = new RegExp(["[À-ß][-¿]", "[à-ï][-¿]{2}", "[ð-÷][-¿]{3}"].join("|"), "g");
var cb_btou = function (cccc) {
switch (cccc.length) {
case 4:
var cp = (7 & cccc.charCodeAt(0)) << 18 | (63 & cccc.charCodeAt(1)) << 12 | (63 & cccc.charCodeAt(2)) << 6 | 63 &
cccc.charCodeAt(3),
offset = cp - 65536;
return fromCharCode((offset >>> 10) + 55296) + fromCharCode((offset & 1023) + 56320);
case 3:
return fromCharCode((15 & cccc.charCodeAt(0)) << 12 | (63 & cccc.charCodeAt(1)) << 6 | 63 & cccc.charCodeAt(2));
default:
return fromCharCode((31 & cccc.charCodeAt(0)) << 6 | 63 & cccc.charCodeAt(1))
}
};
var btou = function (b) {
return b.replace(re_btou, cb_btou)
};
var cb_decode = function (cccc) {
var len = cccc.length,
padlen = len % 4,
n = (len > 0 ? b64tab[cccc.charAt(0)] << 18 : 0) | (len > 1 ? b64tab[cccc.charAt(1)] << 12 : 0) | (len > 2 ?
b64tab[cccc.charAt(2)] << 6 : 0) | (len > 3 ? b64tab[cccc.charAt(3)] : 0),
chars = [fromCharCode(n >>> 16), fromCharCode(n >>> 8 & 255), fromCharCode(n & 255)];
chars.length -= [0, 0, 2, 1][padlen];
return chars.join("")
};
var atob = global.atob ? function (a) {
return global.atob(a)
} : function (a) {
return a.replace(/[\s\S]{1,4}/g, cb_decode)
};
var _decode = buffer ? function (a) {
return (a.constructor === buffer.constructor ? a : new buffer(a, "base64")).toString()
} : function (a) {
return btou(atob(a))
};
var decode = function (a) {
return _decode(String(a).replace(/[-_]/g, function (m0) {
return m0 == "-" ? "+" : "/"
}).replace(/[^A-Za-z0-9\+\/]/g, ""))
};
var noConflict = function () {
var Base64 = global.Base64;
global.Base64 = _Base64;
return Base64
};
global.Base64 = {
VERSION: version,
atob: atob,
btoa: btoa,
fromBase64: decode,
toBase64: encode,
utob: utob,
encode: encode,
encodeURI: encodeURI,
btou: btou,
decode: decode,
noConflict: noConflict
};
if (typeof Object.defineProperty === "function") {
var noEnum = function (v) {
return {
value: v,
enumerable: false,
writable: true,
configurable: true
}
};
global.Base64.extendString = function () {
Object.defineProperty(String.prototype, "fromBase64", noEnum(function () {
return decode(this)
}));
Object.defineProperty(String.prototype, "toBase64", noEnum(function (urisafe) {
return encode(this, urisafe)
}));
Object.defineProperty(String.prototype, "toBase64URI", noEnum(function () {
return encode(this, true)
}))
}
}
if (global["Meteor"]) {
Base64 = global.Base64
}
})(this);
function getEncryption(value) {
var val = Base64.encode(value);
var arr = [];
for (var i = 0; i < val.length; i++) {
arr.push(val.charAt(i));
if (i % 2) {
var num1 = Math.floor(10 * Math.random());
arr.push(num1);
var num2 = Math.floor(10 * Math.random());
arr.push(num2);
var num3 = Math.floor(10 * Math.random());
arr.push(num3)
}
}
return arr.join("");
};
================================================
FILE: 其他实战/【金逸电影】自动注册/register.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-09-04 Python: 3.7
import requests
import execjs.runtime_names
class JinYiRegister:
"""
金逸电影注册
http://www.jycinema.com/wap/#/register
"""
def __init__(self, phone):
self.headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
}
self.url = 'http://www.jycinema.com/frontUIWebapp/appserver/photoMessageService/newsSendMessage'
self.phone = phone
@staticmethod
def js_make(json_data):
with open('encryp.js', 'r', encoding='utf-8') as f:
js = execjs.compile(f.read())
try:
result = js.call("getEncryption", json_data)
return result
except Exception:
print('js 异常')
def register(self):
data = '{"mobileNumber": ' + self.phone + ', "channelId": 7, "channelCode": "J0005", "memberId": ""}'
data = {
'params': self.js_make(data),
'Origin': 'http://www.jycinema.com',
'Referer': 'http://www.jycinema.com/wap/',
}
response = requests.post(self.url, data=data, headers=self.headers)
print(response.content.decode('utf-8'))
if __name__ == '__main__':
your_phone = input('请输入待注册手机号')
jy = JinYiRegister(your_phone)
jy.register()
================================================
FILE: 其他实战/【青海移动】登陆参数生成/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-06 Python: 3.7
================================================
FILE: 其他实战/【青海移动】登陆参数生成/encryp.js
================================================
var CryptoJS = CryptoJS || function (u, l) {
var d = {},
n = d.lib = {},
p = function () {
},
s = n.Base = {
extend: function (a) {
p.prototype = this;
var c = new p;
a && c.mixIn(a);
c.hasOwnProperty("init") || (c.init = function () {
c.$super.init.apply(this, arguments)
});
c.init.prototype = c;
c.$super = this;
return c
},
create: function () {
var a = this.extend();
a.init.apply(a, arguments);
return a
},
init: function () {
},
mixIn: function (a) {
for (var c in a) a.hasOwnProperty(c) && (this[c] = a[c]);
a.hasOwnProperty("toString") && (this.toString = a.toString)
},
clone: function () {
return this.init.prototype.extend(this)
}
},
q = n.WordArray = s.extend({
init: function (a, c) {
a = this.words = a || [];
this.sigBytes = c != l ? c : 4 * a.length
},
toString: function (a) {
return (a || v).stringify(this)
},
concat: function (a) {
var c = this.words,
m = a.words,
f = this.sigBytes;
a = a.sigBytes;
this.clamp();
if (f % 4)
for (var t = 0; t < a; t++) c[f + t >>> 2] |= (m[t >>> 2] >>> 24 - 8 * (t % 4) & 255) << 24 - 8 * ((f + t) %
4);
else if (65535 < m.length)
for (t = 0; t < a; t += 4) c[f + t >>> 2] = m[t >>> 2];
else c.push.apply(c, m);
this.sigBytes += a;
return this
},
clamp: function () {
var a = this.words,
c = this.sigBytes;
a[c >>> 2] &= 4294967295 << 32 - 8 * (c % 4);
a.length = u.ceil(c / 4)
},
clone: function () {
var a = s.clone.call(this);
a.words = this.words.slice(0);
return a
},
random: function (a) {
for (var c = [], m = 0; m < a; m += 4) c.push(4294967296 * u.random() | 0);
return new q.init(c, a)
}
}),
w = d.enc = {},
v = w.Hex = {
stringify: function (a) {
var c = a.words;
a = a.sigBytes;
for (var m = [], f = 0; f < a; f++) {
var t = c[f >>> 2] >>> 24 - 8 * (f % 4) & 255;
m.push((t >>> 4).toString(16));
m.push((t & 15).toString(16))
}
return m.join("")
},
parse: function (a) {
for (var c = a.length, m = [], f = 0; f < c; f += 2) m[f >>> 3] |= parseInt(a.substr(f, 2), 16) << 24 - 4 * (f %
8);
return new q.init(m, c / 2)
}
},
b = w.Latin1 = {
stringify: function (a) {
var c = a.words;
a = a.sigBytes;
for (var m = [], f = 0; f < a; f++) m.push(String.fromCharCode(c[f >>> 2] >>> 24 - 8 * (f % 4) & 255));
return m.join("")
},
parse: function (a) {
for (var c = a.length, m = [], f = 0; f < c; f++) m[f >>> 2] |= (a.charCodeAt(f) & 255) << 24 - 8 * (f % 4);
return new q.init(m, c)
}
},
x = w.Utf8 = {
stringify: function (a) {
try {
return decodeURIComponent(escape(b.stringify(a)))
} catch (c) {
throw Error("Malformed UTF-8 data");
}
},
parse: function (a) {
return b.parse(unescape(encodeURIComponent(a)))
}
},
r = n.BufferedBlockAlgorithm = s.extend({
reset: function () {
this._data = new q.init;
this._nDataBytes = 0
},
_append: function (a) {
"string" == typeof a && (a = x.parse(a));
this._data.concat(a);
this._nDataBytes += a.sigBytes
},
_process: function (a) {
var c = this._data,
m = c.words,
f = c.sigBytes,
t = this.blockSize,
b = f / (4 * t),
b = a ? u.ceil(b) : u.max((b | 0) - this._minBufferSize, 0);
a = b * t;
f = u.min(4 * a, f);
if (a) {
for (var e = 0; e < a; e += t) this._doProcessBlock(m, e);
e = m.splice(0, a);
c.sigBytes -= f
}
return new q.init(e, f)
},
clone: function () {
var a = s.clone.call(this);
a._data = this._data.clone();
return a
},
_minBufferSize: 0
});
n.Hasher = r.extend({
cfg: s.extend(),
init: function (a) {
this.cfg = this.cfg.extend(a);
this.reset()
},
reset: function () {
r.reset.call(this);
this._doReset()
},
update: function (a) {
this._append(a);
this._process();
return this
},
finalize: function (a) {
a && this._append(a);
return this._doFinalize()
},
blockSize: 16,
_createHelper: function (a) {
return function (c, m) {
return (new a.init(m)).finalize(c)
}
},
_createHmacHelper: function (a) {
return function (c, m) {
return (new e.HMAC.init(a, m)).finalize(c)
}
}
});
var e = d.algo = {};
return d
}(Math);
(function () {
var u = CryptoJS,
l = u.lib.WordArray;
u.enc.Base64 = {
stringify: function (d) {
var n = d.words,
l = d.sigBytes,
s = this._map;
d.clamp();
d = [];
for (var q = 0; q < l; q += 3)
for (var w = (n[q >>> 2] >>> 24 - 8 * (q % 4) & 255) << 16 | (n[q + 1 >>> 2] >>> 24 - 8 * ((q + 1) % 4) & 255) <<
8 | n[q + 2 >>> 2] >>> 24 - 8 * ((q + 2) % 4) & 255, v = 0; 4 > v && q + 0.75 * v < l; v++) d.push(s.charAt(
w >>> 6 * (3 - v) & 63));
if (n = s.charAt(64))
for (; d.length % 4;) d.push(n);
return d.join("")
},
parse: function (d) {
var n = d.length,
p = this._map,
s = p.charAt(64);
s && (s = d.indexOf(s), -1 != s && (n = s));
for (var s = [], q = 0, w = 0; w < n; w++)
if (w % 4) {
var v = p.indexOf(d.charAt(w - 1)) << 2 * (w % 4),
b = p.indexOf(d.charAt(w)) >>> 6 - 2 * (w % 4);
s[q >>> 2] |= (v | b) << 24 - 8 * (q % 4);
q++
}
return l.create(s, q)
},
_map: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
}
})();
(function (u) {
function l(b, e, a, c, m, f, t) {
b = b + (e & a | ~e & c) + m + t;
return (b << f | b >>> 32 - f) + e
}
function d(b, e, a, c, m, f, t) {
b = b + (e & c | a & ~c) + m + t;
return (b << f | b >>> 32 - f) + e
}
function n(b, e, a, c, m, f, t) {
b = b + (e ^ a ^ c) + m + t;
return (b << f | b >>> 32 - f) + e
}
function p(b, e, a, c, m, f, t) {
b = b + (a ^ (e | ~c)) + m + t;
return (b << f | b >>> 32 - f) + e
}
for (var s = CryptoJS, q = s.lib, w = q.WordArray, v = q.Hasher, q = s.algo, b = [], x = 0; 64 > x; x++) b[x] =
4294967296 * u.abs(u.sin(x + 1)) | 0;
q = q.MD5 = v.extend({
_doReset: function () {
this._hash = new w.init([1732584193, 4023233417, 2562383102, 271733878])
},
_doProcessBlock: function (r, e) {
for (var a = 0; 16 > a; a++) {
var c = e + a,
m = r[c];
r[c] = (m << 8 | m >>> 24) & 16711935 | (m << 24 | m >>> 8) & 4278255360
}
var a = this._hash.words,
c = r[e + 0],
m = r[e + 1],
f = r[e + 2],
t = r[e + 3],
y = r[e + 4],
q = r[e + 5],
s = r[e + 6],
w = r[e + 7],
v = r[e + 8],
u = r[e + 9],
x = r[e + 10],
z = r[e + 11],
A = r[e + 12],
B = r[e + 13],
C = r[e + 14],
D = r[e + 15],
g = a[0],
h = a[1],
j = a[2],
k = a[3],
g = l(g, h, j, k, c, 7, b[0]),
k = l(k, g, h, j, m, 12, b[1]),
j = l(j, k, g, h, f, 17, b[2]),
h = l(h, j, k, g, t, 22, b[3]),
g = l(g, h, j, k, y, 7, b[4]),
k = l(k, g, h, j, q, 12, b[5]),
j = l(j, k, g, h, s, 17, b[6]),
h = l(h, j, k, g, w, 22, b[7]),
g = l(g, h, j, k, v, 7, b[8]),
k = l(k, g, h, j, u, 12, b[9]),
j = l(j, k, g, h, x, 17, b[10]),
h = l(h, j, k, g, z, 22, b[11]),
g = l(g, h, j, k, A, 7, b[12]),
k = l(k, g, h, j, B, 12, b[13]),
j = l(j, k, g, h, C, 17, b[14]),
h = l(h, j, k, g, D, 22, b[15]),
g = d(g, h, j, k, m, 5, b[16]),
k = d(k, g, h, j, s, 9, b[17]),
j = d(j, k, g, h, z, 14, b[18]),
h = d(h, j, k, g, c, 20, b[19]),
g = d(g, h, j, k, q, 5, b[20]),
k = d(k, g, h, j, x, 9, b[21]),
j = d(j, k, g, h, D, 14, b[22]),
h = d(h, j, k, g, y, 20, b[23]),
g = d(g, h, j, k, u, 5, b[24]),
k = d(k, g, h, j, C, 9, b[25]),
j = d(j, k, g, h, t, 14, b[26]),
h = d(h, j, k, g, v, 20, b[27]),
g = d(g, h, j, k, B, 5, b[28]),
k = d(k, g, h, j, f, 9, b[29]),
j = d(j, k, g, h, w, 14, b[30]),
h = d(h, j, k, g, A, 20, b[31]),
g = n(g, h, j, k, q, 4, b[32]),
k = n(k, g, h, j, v, 11, b[33]),
j = n(j, k, g, h, z, 16, b[34]),
h = n(h, j, k, g, C, 23, b[35]),
g = n(g, h, j, k, m, 4, b[36]),
k = n(k, g, h, j, y, 11, b[37]),
j = n(j, k, g, h, w, 16, b[38]),
h = n(h, j, k, g, x, 23, b[39]),
g = n(g, h, j, k, B, 4, b[40]),
k = n(k, g, h, j, c, 11, b[41]),
j = n(j, k, g, h, t, 16, b[42]),
h = n(h, j, k, g, s, 23, b[43]),
g = n(g, h, j, k, u, 4, b[44]),
k = n(k, g, h, j, A, 11, b[45]),
j = n(j, k, g, h, D, 16, b[46]),
h = n(h, j, k, g, f, 23, b[47]),
g = p(g, h, j, k, c, 6, b[48]),
k = p(k, g, h, j, w, 10, b[49]),
j = p(j, k, g, h, C, 15, b[50]),
h = p(h, j, k, g, q, 21, b[51]),
g = p(g, h, j, k, A, 6, b[52]),
k = p(k, g, h, j, t, 10, b[53]),
j = p(j, k, g, h, x, 15, b[54]),
h = p(h, j, k, g, m, 21, b[55]),
g = p(g, h, j, k, v, 6, b[56]),
k = p(k, g, h, j, D, 10, b[57]),
j = p(j, k, g, h, s, 15, b[58]),
h = p(h, j, k, g, B, 21, b[59]),
g = p(g, h, j, k, y, 6, b[60]),
k = p(k, g, h, j, z, 10, b[61]),
j = p(j, k, g, h, f, 15, b[62]),
h = p(h, j, k, g, u, 21, b[63]);
a[0] = a[0] + g | 0;
a[1] = a[1] + h | 0;
a[2] = a[2] + j | 0;
a[3] = a[3] + k | 0
},
_doFinalize: function () {
var b = this._data,
e = b.words,
a = 8 * this._nDataBytes,
c = 8 * b.sigBytes;
e[c >>> 5] |= 128 << 24 - c % 32;
var m = u.floor(a / 4294967296);
e[(c + 64 >>> 9 << 4) + 15] = (m << 8 | m >>> 24) & 16711935 | (m << 24 | m >>> 8) & 4278255360;
e[(c + 64 >>> 9 << 4) + 14] = (a << 8 | a >>> 24) & 16711935 | (a << 24 | a >>> 8) & 4278255360;
b.sigBytes = 4 * (e.length + 1);
this._process();
b = this._hash;
e = b.words;
for (a = 0; 4 > a; a++) c = e[a], e[a] = (c << 8 | c >>> 24) & 16711935 | (c << 24 | c >>> 8) & 4278255360;
return b
},
clone: function () {
var b = v.clone.call(this);
b._hash = this._hash.clone();
return b
}
});
s.MD5 = v._createHelper(q);
s.HmacMD5 = v._createHmacHelper(q)
})(Math);
(function () {
var u = CryptoJS,
l = u.lib,
d = l.Base,
n = l.WordArray,
l = u.algo,
p = l.EvpKDF = d.extend({
cfg: d.extend({
keySize: 4,
hasher: l.MD5,
iterations: 1
}),
init: function (d) {
this.cfg = this.cfg.extend(d)
},
compute: function (d, l) {
for (var p = this.cfg, v = p.hasher.create(), b = n.create(), u = b.words, r = p.keySize, p = p.iterations; u
.length < r;) {
e && v.update(e);
var e = v.update(d).finalize(l);
v.reset();
for (var a = 1; a < p; a++) e = v.finalize(e), v.reset();
b.concat(e)
}
b.sigBytes = 4 * r;
return b
}
});
u.EvpKDF = function (d, l, n) {
return p.create(n).compute(d, l)
}
})();
CryptoJS.lib.Cipher || function (u) {
var l = CryptoJS,
d = l.lib,
n = d.Base,
p = d.WordArray,
s = d.BufferedBlockAlgorithm,
q = l.enc.Base64,
w = l.algo.EvpKDF,
v = d.Cipher = s.extend({
cfg: n.extend(),
createEncryptor: function (m, a) {
return this.create(this._ENC_XFORM_MODE, m, a)
},
createDecryptor: function (m, a) {
return this.create(this._DEC_XFORM_MODE, m, a)
},
init: function (m, a, b) {
this.cfg = this.cfg.extend(b);
this._xformMode = m;
this._key = a;
this.reset()
},
reset: function () {
s.reset.call(this);
this._doReset()
},
process: function (a) {
this._append(a);
return this._process()
},
finalize: function (a) {
a && this._append(a);
return this._doFinalize()
},
keySize: 4,
ivSize: 4,
_ENC_XFORM_MODE: 1,
_DEC_XFORM_MODE: 2,
_createHelper: function (m) {
return {
encrypt: function (f, b, e) {
return ("string" == typeof b ? c : a).encrypt(m, f, b, e)
},
decrypt: function (f, b, e) {
return ("string" == typeof b ? c : a).decrypt(m, f, b, e)
}
}
}
});
d.StreamCipher = v.extend({
_doFinalize: function () {
return this._process(!0)
},
blockSize: 1
});
var b = l.mode = {},
x = function (a, f, b) {
var c = this._iv;
c ? this._iv = u : c = this._prevBlock;
for (var e = 0; e < b; e++) a[f + e] ^= c[e]
},
r = (d.BlockCipherMode = n.extend({
createEncryptor: function (a, f) {
return this.Encryptor.create(a, f)
},
createDecryptor: function (a, f) {
return this.Decryptor.create(a, f)
},
init: function (a, f) {
this._cipher = a;
this._iv = f
}
})).extend();
r.Encryptor = r.extend({
processBlock: function (a, f) {
var b = this._cipher,
c = b.blockSize;
x.call(this, a, f, c);
b.encryptBlock(a, f);
this._prevBlock = a.slice(f, f + c)
}
});
r.Decryptor = r.extend({
processBlock: function (a, b) {
var c = this._cipher,
e = c.blockSize,
d = a.slice(b, b + e);
c.decryptBlock(a, b);
x.call(this, a, b, e);
this._prevBlock = d
}
});
b = b.CBC = r;
r = (l.pad = {}).Pkcs7 = {
pad: function (a, b) {
for (var c = 4 * b, c = c - a.sigBytes % c, e = c << 24 | c << 16 | c << 8 | c, d = [], l = 0; l < c; l += 4) d
.push(e);
c = p.create(d, c);
a.concat(c)
},
unpad: function (a) {
a.sigBytes -= a.words[a.sigBytes - 1 >>> 2] & 255
}
};
d.BlockCipher = v.extend({
cfg: v.cfg.extend({
mode: b,
padding: r
}),
reset: function () {
v.reset.call(this);
var a = this.cfg,
c = a.iv,
a = a.mode;
if (this._xformMode == this._ENC_XFORM_MODE) var b = a.createEncryptor;
else b = a.createDecryptor, this._minBufferSize = 1;
this._mode = b.call(a, this, c && c.words)
},
_doProcessBlock: function (a, c) {
this._mode.processBlock(a, c)
},
_doFinalize: function () {
var a = this.cfg.padding;
if (this._xformMode == this._ENC_XFORM_MODE) {
a.pad(this._data, this.blockSize);
var c = this._process(!0)
} else c = this._process(!0), a.unpad(c);
return c
},
blockSize: 4
});
var e = d.CipherParams = n.extend({
init: function (a) {
this.mixIn(a)
},
toString: function (a) {
return (a || this.formatter).stringify(this)
}
}),
b = (l.format = {}).OpenSSL = {
stringify: function (a) {
var c = a.ciphertext;
a = a.salt;
return (a ? p.create([1398893684, 1701076831]).concat(a).concat(c) : c).toString(q)
},
parse: function (a) {
a = q.parse(a);
var c = a.words;
if (1398893684 == c[0] && 1701076831 == c[1]) {
var b = p.create(c.slice(2, 4));
c.splice(0, 4);
a.sigBytes -= 16
}
return e.create({
ciphertext: a,
salt: b
})
}
},
a = d.SerializableCipher = n.extend({
cfg: n.extend({
format: b
}),
encrypt: function (a, c, b, d) {
d = this.cfg.extend(d);
var l = a.createEncryptor(b, d);
c = l.finalize(c);
l = l.cfg;
return e.create({
ciphertext: c,
key: b,
iv: l.iv,
algorithm: a,
mode: l.mode,
padding: l.padding,
blockSize: a.blockSize,
formatter: d.format
})
},
decrypt: function (a, c, b, e) {
e = this.cfg.extend(e);
c = this._parse(c, e.format);
return a.createDecryptor(b, e).finalize(c.ciphertext)
},
_parse: function (a, c) {
return "string" == typeof a ? c.parse(a, this) : a
}
}),
l = (l.kdf = {}).OpenSSL = {
execute: function (a, c, b, d) {
d || (d = p.random(8));
a = w.create({
keySize: c + b
}).compute(a, d);
b = p.create(a.words.slice(c), 4 * b);
a.sigBytes = 4 * c;
return e.create({
key: a,
iv: b,
salt: d
})
}
},
c = d.PasswordBasedCipher = a.extend({
cfg: a.cfg.extend({
kdf: l
}),
encrypt: function (c, b, e, d) {
d = this.cfg.extend(d);
e = d.kdf.execute(e, c.keySize, c.ivSize);
d.iv = e.iv;
c = a.encrypt.call(this, c, b, e.key, d);
c.mixIn(e);
return c
},
decrypt: function (c, b, e, d) {
d = this.cfg.extend(d);
b = this._parse(b, d.format);
e = d.kdf.execute(e, c.keySize, c.ivSize, b.salt);
d.iv = e.iv;
return a.decrypt.call(this, c, b, e.key, d)
}
})
}();
(function () {
function u(b, a) {
var c = (this._lBlock >>> b ^ this._rBlock) & a;
this._rBlock ^= c;
this._lBlock ^= c << b
}
function l(b, a) {
var c = (this._rBlock >>> b ^ this._lBlock) & a;
this._lBlock ^= c;
this._rBlock ^= c << b
}
var d = CryptoJS,
n = d.lib,
p = n.WordArray,
n = n.BlockCipher,
s = d.algo,
q = [57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, 27, 19, 11, 3, 60, 52, 44, 36,
63, 55, 47, 39, 31, 23, 15, 7, 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4
],
w = [14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47,
55, 30, 40, 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32
],
v = [1, 2, 4, 6, 8, 10, 12, 14, 15, 17, 19, 21, 23, 25, 27, 28],
b = [{
"0": 8421888,
268435456: 32768,
536870912: 8421378,
805306368: 2,
1073741824: 512,
1342177280: 8421890,
1610612736: 8389122,
1879048192: 8388608,
2147483648: 514,
2415919104: 8389120,
2684354560: 33280,
2952790016: 8421376,
3221225472: 32770,
3489660928: 8388610,
3758096384: 0,
4026531840: 33282,
134217728: 0,
402653184: 8421890,
671088640: 33282,
939524096: 32768,
1207959552: 8421888,
1476395008: 512,
1744830464: 8421378,
2013265920: 2,
2281701376: 8389120,
2550136832: 33280,
2818572288: 8421376,
3087007744: 8389122,
3355443200: 8388610,
3623878656: 32770,
3892314112: 514,
4160749568: 8388608,
1: 32768,
268435457: 2,
536870913: 8421888,
805306369: 8388608,
1073741825: 8421378,
1342177281: 33280,
1610612737: 512,
1879048193: 8389122,
2147483649: 8421890,
2415919105: 8421376,
2684354561: 8388610,
2952790017: 33282,
3221225473: 514,
3489660929: 8389120,
3758096385: 32770,
4026531841: 0,
134217729: 8421890,
402653185: 8421376,
671088641: 8388608,
939524097: 512,
1207959553: 32768,
1476395009: 8388610,
1744830465: 2,
2013265921: 33282,
2281701377: 32770,
2550136833: 8389122,
2818572289: 514,
3087007745: 8421888,
3355443201: 8389120,
3623878657: 0,
3892314113: 33280,
4160749569: 8421378
}, {
"0": 1074282512,
16777216: 16384,
33554432: 524288,
50331648: 1074266128,
67108864: 1073741840,
83886080: 1074282496,
100663296: 1073758208,
117440512: 16,
134217728: 540672,
150994944: 1073758224,
167772160: 1073741824,
184549376: 540688,
201326592: 524304,
218103808: 0,
234881024: 16400,
251658240: 1074266112,
8388608: 1073758208,
25165824: 540688,
41943040: 16,
58720256: 1073758224,
75497472: 1074282512,
92274688: 1073741824,
109051904: 524288,
125829120: 1074266128,
142606336: 524304,
159383552: 0,
176160768: 16384,
192937984: 1074266112,
209715200: 1073741840,
226492416: 540672,
243269632: 1074282496,
260046848: 16400,
268435456: 0,
285212672: 1074266128,
301989888: 1073758224,
318767104: 1074282496,
335544320: 1074266112,
352321536: 16,
369098752: 540688,
385875968: 16384,
402653184: 16400,
419430400: 524288,
436207616: 524304,
452984832: 1073741840,
469762048: 540672,
486539264: 1073758208,
503316480: 1073741824,
520093696: 1074282512,
276824064: 540688,
293601280: 524288,
310378496: 1074266112,
327155712: 16384,
343932928: 1073758208,
360710144: 1074282512,
377487360: 16,
394264576: 1073741824,
411041792: 1074282496,
427819008: 1073741840,
444596224: 1073758224,
461373440: 524304,
478150656: 0,
494927872: 16400,
511705088: 1074266128,
528482304: 540672
}, {
"0": 260,
1048576: 0,
2097152: 67109120,
3145728: 65796,
4194304: 65540,
5242880: 67108868,
6291456: 67174660,
7340032: 67174400,
8388608: 67108864,
9437184: 67174656,
10485760: 65792,
11534336: 67174404,
12582912: 67109124,
13631488: 65536,
14680064: 4,
15728640: 256,
524288: 67174656,
1572864: 67174404,
2621440: 0,
3670016: 67109120,
4718592: 67108868,
5767168: 65536,
6815744: 65540,
7864320: 260,
8912896: 4,
9961472: 256,
11010048: 67174400,
12058624: 65796,
13107200: 65792,
14155776: 67109124,
15204352: 67174660,
16252928: 67108864,
16777216: 67174656,
17825792: 65540,
18874368: 65536,
19922944: 67109120,
20971520: 256,
22020096: 67174660,
23068672: 67108868,
24117248: 0,
25165824: 67109124,
26214400: 67108864,
27262976: 4,
28311552: 65792,
29360128: 67174400,
30408704: 260,
31457280: 65796,
32505856: 67174404,
17301504: 67108864,
18350080: 260,
19398656: 67174656,
20447232: 0,
21495808: 65540,
22544384: 67109120,
23592960: 256,
24641536: 67174404,
25690112: 65536,
26738688: 67174660,
27787264: 65796,
28835840: 67108868,
29884416: 67109124,
30932992: 67174400,
31981568: 4,
33030144: 65792
}, {
"0": 2151682048,
65536: 2147487808,
131072: 4198464,
196608: 2151677952,
262144: 0,
327680: 4198400,
393216: 2147483712,
458752: 4194368,
524288: 2147483648,
589824: 4194304,
655360: 64,
720896: 2147487744,
786432: 2151678016,
851968: 4160,
917504: 4096,
983040: 2151682112,
32768: 2147487808,
98304: 64,
163840: 2151678016,
229376: 2147487744,
294912: 4198400,
360448: 2151682112,
425984: 0,
491520: 2151677952,
557056: 4096,
622592: 2151682048,
688128: 4194304,
753664: 4160,
819200: 2147483648,
884736: 4194368,
950272: 4198464,
1015808: 2147483712,
1048576: 4194368,
1114112: 4198400,
1179648: 2147483712,
1245184: 0,
1310720: 4160,
1376256: 2151678016,
1441792: 2151682048,
1507328: 2147487808,
1572864: 2151682112,
1638400: 2147483648,
1703936: 2151677952,
1769472: 4198464,
1835008: 2147487744,
1900544: 4194304,
1966080: 64,
2031616: 4096,
1081344: 2151677952,
1146880: 2151682112,
1212416: 0,
1277952: 4198400,
1343488: 4194368,
1409024: 2147483648,
1474560: 2147487808,
1540096: 64,
1605632: 2147483712,
1671168: 4096,
1736704: 2147487744,
1802240: 2151678016,
1867776: 4160,
1933312: 2151682048,
1998848: 4194304,
2064384: 4198464
}, {
"0": 128,
4096: 17039360,
8192: 262144,
12288: 536870912,
16384: 537133184,
20480: 16777344,
24576: 553648256,
28672: 262272,
32768: 16777216,
36864: 537133056,
40960: 536871040,
45056: 553910400,
49152: 553910272,
53248: 0,
57344: 17039488,
61440: 553648128,
2048: 17039488,
6144: 553648256,
10240: 128,
14336: 17039360,
18432: 262144,
22528: 537133184,
26624: 553910272,
30720: 536870912,
34816: 537133056,
38912: 0,
43008: 553910400,
47104: 16777344,
51200: 536871040,
55296: 553648128,
59392: 16777216,
63488: 262272,
65536: 262144,
69632: 128,
73728: 536870912,
77824: 553648256,
81920: 16777344,
86016: 553910272,
90112: 537133184,
94208: 16777216,
98304: 553910400,
102400: 553648128,
106496: 17039360,
110592: 537133056,
114688: 262272,
118784: 536871040,
122880: 0,
126976: 17039488,
67584: 553648256,
71680: 16777216,
75776: 17039360,
79872: 537133184,
83968: 536870912,
88064: 17039488,
92160: 128,
96256: 553910272,
100352: 262272,
104448: 553910400,
108544: 0,
112640: 553648128,
116736: 16777344,
120832: 262144,
124928: 537133056,
129024: 536871040
}, {
"0": 268435464,
256: 8192,
512: 270532608,
768: 270540808,
1024: 268443648,
1280: 2097152,
1536: 2097160,
1792: 268435456,
2048: 0,
2304: 268443656,
2560: 2105344,
2816: 8,
3072: 270532616,
3328: 2105352,
3584: 8200,
3840: 270540800,
128: 270532608,
384: 270540808,
640: 8,
896: 2097152,
1152: 2105352,
1408: 268435464,
1664: 268443648,
1920: 8200,
2176: 2097160,
2432: 8192,
2688: 268443656,
2944: 270532616,
3200: 0,
3456: 270540800,
3712: 2105344,
3968: 268435456,
4096: 268443648,
4352: 270532616,
4608: 270540808,
4864: 8200,
5120: 2097152,
5376: 268435456,
5632: 268435464,
5888: 2105344,
6144: 2105352,
6400: 0,
6656: 8,
6912: 270532608,
7168: 8192,
7424: 268443656,
7680: 270540800,
7936: 2097160,
4224: 8,
4480: 2105344,
4736: 2097152,
4992: 268435464,
5248: 268443648,
5504: 8200,
5760: 270540808,
6016: 270532608,
6272: 270540800,
6528: 270532616,
6784: 8192,
7040: 2105352,
7296: 2097160,
7552: 0,
7808: 268435456,
8064: 268443656
}, {
"0": 1048576,
16: 33555457,
32: 1024,
48: 1049601,
64: 34604033,
80: 0,
96: 1,
112: 34603009,
128: 33555456,
144: 1048577,
160: 33554433,
176: 34604032,
192: 34603008,
208: 1025,
224: 1049600,
240: 33554432,
8: 34603009,
24: 0,
40: 33555457,
56: 34604032,
72: 1048576,
88: 33554433,
104: 33554432,
120: 1025,
136: 1049601,
152: 33555456,
168: 34603008,
184: 1048577,
200: 1024,
216: 34604033,
232: 1,
248: 1049600,
256: 33554432,
272: 1048576,
288: 33555457,
304: 34603009,
320: 1048577,
336: 33555456,
352: 34604032,
368: 1049601,
384: 1025,
400: 34604033,
416: 1049600,
432: 1,
448: 0,
464: 34603008,
480: 33554433,
496: 1024,
264: 1049600,
280: 33555457,
296: 34603009,
312: 1,
328: 33554432,
344: 1048576,
360: 1025,
376: 34604032,
392: 33554433,
408: 34603008,
424: 0,
440: 34604033,
456: 1049601,
472: 1024,
488: 33555456,
504: 1048577
}, {
"0": 134219808,
1: 131072,
2: 134217728,
3: 32,
4: 131104,
5: 134350880,
6: 134350848,
7: 2048,
8: 134348800,
9: 134219776,
10: 133120,
11: 134348832,
12: 2080,
13: 0,
14: 134217760,
15: 133152,
2147483648: 2048,
2147483649: 134350880,
2147483650: 134219808,
2147483651: 134217728,
2147483652: 134348800,
2147483653: 133120,
2147483654: 133152,
2147483655: 32,
2147483656: 134217760,
2147483657: 2080,
2147483658: 131104,
2147483659: 134350848,
2147483660: 0,
2147483661: 134348832,
2147483662: 134219776,
2147483663: 131072,
16: 133152,
17: 134350848,
18: 32,
19: 2048,
20: 134219776,
21: 134217760,
22: 134348832,
23: 131072,
24: 0,
25: 131104,
26: 134348800,
27: 134219808,
28: 134350880,
29: 133120,
30: 2080,
31: 134217728,
2147483664: 131072,
2147483665: 2048,
2147483666: 134348832,
2147483667: 133152,
2147483668: 32,
2147483669: 134348800,
2147483670: 134217728,
2147483671: 134219808,
2147483672: 134350880,
2147483673: 134217760,
2147483674: 134219776,
2147483675: 0,
2147483676: 133120,
2147483677: 2080,
2147483678: 131104,
2147483679: 134350848
}],
x = [4160749569, 528482304, 33030144, 2064384, 129024, 8064, 504, 2147483679],
r = s.DES = n.extend({
_doReset: function () {
for (var b = this._key.words, a = [], c = 0; 56 > c; c++) {
var d = q[c] - 1;
a[c] = b[d >>> 5] >>> 31 - d % 32 & 1
}
b = this._subKeys = [];
for (d = 0; 16 > d; d++) {
for (var f = b[d] = [], l = v[d], c = 0; 24 > c; c++) f[c / 6 | 0] |= a[(w[c] - 1 + l) % 28] << 31 - c % 6,
f[4 + (c / 6 | 0)] |= a[28 + (w[c + 24] - 1 + l) % 28] << 31 - c % 6;
f[0] = f[0] << 1 | f[0] >>> 31;
for (c = 1; 7 > c; c++) f[c] >>>= 4 * (c - 1) + 3;
f[7] = f[7] << 5 | f[7] >>> 27
}
a = this._invSubKeys = [];
for (c = 0; 16 > c; c++) a[c] = b[15 - c]
},
encryptBlock: function (b, a) {
this._doCryptBlock(b, a, this._subKeys)
},
decryptBlock: function (b, a) {
this._doCryptBlock(b, a, this._invSubKeys)
},
_doCryptBlock: function (e, a, c) {
this._lBlock = e[a];
this._rBlock = e[a + 1];
u.call(this, 4, 252645135);
u.call(this, 16, 65535);
l.call(this, 2, 858993459);
l.call(this, 8, 16711935);
u.call(this, 1, 1431655765);
for (var d = 0; 16 > d; d++) {
for (var f = c[d], n = this._lBlock, p = this._rBlock, q = 0, r = 0; 8 > r; r++) q |= b[r][((p ^ f[r]) & x[r]) >>>
0
];
this._lBlock = p;
this._rBlock = n ^ q
}
c = this._lBlock;
this._lBlock = this._rBlock;
this._rBlock = c;
u.call(this, 1, 1431655765);
l.call(this, 8, 16711935);
l.call(this, 2, 858993459);
u.call(this, 16, 65535);
u.call(this, 4, 252645135);
e[a] = this._lBlock;
e[a + 1] = this._rBlock
},
keySize: 2,
ivSize: 2,
blockSize: 2
});
d.DES = n._createHelper(r);
s = s.TripleDES = n.extend({
_doReset: function () {
var b = this._key.words;
this._des1 = r.createEncryptor(p.create(b.slice(0, 2)));
this._des2 = r.createEncryptor(p.create(b.slice(2, 4)));
this._des3 = r.createEncryptor(p.create(b.slice(4, 6)))
},
encryptBlock: function (b, a) {
this._des1.encryptBlock(b, a);
this._des2.decryptBlock(b, a);
this._des3.encryptBlock(b, a)
},
decryptBlock: function (b, a) {
this._des3.decryptBlock(b, a);
this._des2.encryptBlock(b, a);
this._des1.decryptBlock(b, a)
},
keySize: 6,
ivSize: 2,
blockSize: 2
});
d.TripleDES = n._createHelper(s)
})();
CryptoJS.mode.ECB = (function () {
var ECB = CryptoJS.lib.BlockCipherMode.extend();
ECB.Encryptor = ECB.extend({
processBlock: function (words, offset) {
this._cipher.encryptBlock(words, offset);
}
});
ECB.Decryptor = ECB.extend({
processBlock: function (words, offset) {
this._cipher.decryptBlock(words, offset);
}
});
return ECB;
}());
var decryptRSAKey;
//rsa解密 --privateKeyStr解密key ，cKeyStr要解密数据
function decryptRSA(privateKeyStr, cKeyStr) {
var crypt = new JSEncrypt();
crypt.setPrivateKey(privateKeyStr);
decryptRSAKey = crypt.decrypt(cKeyStr);
return decryptRSAKey;
}
//DES加密 message要加密数据，key加密key
function encryptByDES(message) {
var keyHex = CryptoJS.enc.Utf8.parse(decryptRSAKey);
var encrypted = CryptoJS.DES.encrypt(message, keyHex, {
mode: CryptoJS.mode.ECB,
padding: CryptoJS.pad.Pkcs7
});
return encrypted.toString();
}
function decryptByDES(ciphertext) {
var keyHex = CryptoJS.enc.Utf8.parse(decryptRSAKey);
var decrypted = CryptoJS.DES.decrypt({
ciphertext: CryptoJS.enc.Base64.parse(ciphertext)
}, keyHex, {
mode: CryptoJS.mode.ECB,
padding: CryptoJS.pad.Pkcs7
});
return decrypted.toString(CryptoJS.enc.Utf8);
}
function test(key) {
return encryptByDES(key)
}
================================================
FILE: 其他实战/【青海移动】登陆参数生成/make_param.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-09-12 Python: 3.7
import execjs.runtime_names
class QinHaiYiDong:
"""
青海移动
参数加密
https://www.iqhmall.cn/shopweb/logon/logon
"""
def __init__(self, user, pwd):
self.js = None
self.user = user
self.pwd = pwd
self.init_js()
def init_js(self):
print('引擎', execjs.get().name)
with open("encryp.js", "r", encoding="utf-8") as f:
self.js = execjs.compile(f.read())
def make_param(self):
print(self.js.call('test', self.pwd))
if __name__ == '__main__':
yd = QinHaiYiDong('17327362817', '123123123')
yd.make_param()
================================================
FILE: 其他实战/【餐饮】查询信息/FoodInfo.py
================================================
# -*- encoding: utf-8 -*-
# Time : 2020/01/16
# Author : Zok
# Email : 362416272@qq.com
import requests
import re
import json
from copyheaders import headers_raw_to_dict
class Food:
"""
根据输入美团餐馆名，解析参观基础信息
"""
def __init__(self):
self.headers = headers_raw_to_dict(b"""
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9
Cache-Control: max-age=0
Connection: keep-alive
Cookie: _lxsdk_s=16fb0ce3a0d-4cf-d9e-cf2%7C%7C1
Host: www.meituan.com
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: none
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36
""")
def get_info(self, url):
response = requests.get(url, headers=self.headers)
data = json.loads(re.search(r'北京美团网-北京美食_酒店_旅游_团购_电影_吃喝玩乐
抱歉，页面暂时无法访问...
错误信息:服务器拒绝请求
返回美团首页
# 可拓展式解密器
[**博客传送门**](https://blog.zhangkunzhi.com/2019/06/02/%E5%8E%9F%E5%88%9B%E5%B7%A5%E5%85%B7%E4%B9%8B%E5%8F%AF%E6%8B%93%E5%B1%95%E8%A7%A3%E7%A0%81%E5%99%A8/index.html)
> 方便测试可连续转换重制的编码转换器，可灵活拓展解码规则
# 代理池清洗工具
[**博客传送门**](https://blog.zhangkunzhi.com/2019/05/02/%E6%90%AD%E5%BB%BA%E4%B8%80%E4%B8%AA%E8%B6%85%E7%AE%80%E5%8D%95%E7%9A%84%E5%AE%9E%E7%94%A8%E7%9A%84%E9%AB%98%E5%8F%AF%E7%94%A8%E4%BB%98%E8%B4%B9IP%E6%B1%A0/index.html)
> 爬虫经常会用到代理ip，其中有很多收费ip，但是如何在scrapy中，高效使用这些ip是一个比较麻烦的事情，在这里基于[芝麻代理ip](http://h.zhimaruanjian.com/pay/)做一个代理池监控器，首先整理我们的需求再对其代理质量进行管理，从而保持高效IP使用率
# 验证码短信接收器
> 基于短信接收平台的异步短信接收器，最大并发上限 20，Python3.5+。
启动后会根据设置的异步并发数进行获取手机号码并监听短信接收情况（60秒） 超过60秒后会将未收到短信的手机号拉入黑名单，并是释放。
若要配置具体某个网站使用，还需开发对应的账号注册器，配合调用本短信接收器来达到自动注册账号的功能
# cookies获取Demo
> 基于Pyppeteer 并发获取站点cookies
- 美团登陆cookies
# 跨数据库迁移器
**工作中经常有这种需求**
> 将采集好的mongodb数据转存到mysql中，或者是redis数据转到mongodb，于是打算封装一个组件便于以后调用
- [x] mysql 数据迁移 mongodb
================================================
FILE: 原创爬虫工具/Register/MessageCode.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-05-13 Python: 3.7
import requests
import json
import time
import asyncio
import aiohttp
ERROR = {
'1001': '参数token不能为空',
'1002': '参数action不能为空',
'1003': '参数action错误',
'1004': 'token失效',
'1005': '用户名或密码错误',
'1006': '用户名不能为空',
'1007': '密码不能为空',
'1008': '账户余额不足',
'1009': '账户被禁用',
'1010': '参数错误',
'1011': '账户待审核',
'1012': '账户暂停使用',
'1013': '接口功能未开启',
'1014': '接口登录未开启',
'2001': '参数itemid不能为空',
'2002': '项目不存在',
'2003': '项目未启用',
'2004': '暂时没有可用的号码',
'2005': '获取号码数量已达到上限',
'2006': '参数mobile不能为空',
'2007': '号码已被释放',
'2008': '号码已离线',
'2009': '发送内容不能为空',
'2010': '号码正在使用中',
'3001': '尚未收到短信',
'3002': '等待发送',
'3003': '正在发送',
'3004': '发送失败',
'3005': '订单不存在',
'9001': '系统错误',
'9002': '系统异常',
}
class YMReg(object):
def __init__(self, token, project_code, aio_count=20):
"""
:param token: 账号 token，在官网后台查询
:param project_code: 项目编码，官网查询
:param aio_count: 并发上限默认 20 (官方)
"""
self.token = token
self.project_code = project_code
self.aio_count = aio_count
print(self.get_balance) # 打印余额
@property
def get_balance(self):
"""查询余额
"""
balance_api = 'http://api.fxhyd.cn/UserInterface.aspx?action=getaccountinfo&token={token}&format=1'
target_url = balance_api.format(token=self.token)
response = requests.get(target_url)
if response.status_code == 200:
state, data = response.text.split('|')
return '当前余额： %s 元' % json.loads(data).get('Balance')
else:
return '获取失败请检测账号'
@ staticmethod
def filter_info(message):
if message in ERROR:
return ERROR.get('message')
elif len(message.split('|')) == 2:
return message.split('|')[1] # 返回正确信息
elif message == 'success':
return '拉黑或释放成功'
else:
return '未知错误'
async def get_node(self, session, phone):
"""接收短信
"""
for i in range(12):
await asyncio.sleep(5)
node_api = 'http://api.fxhyd.cn/UserInterface.aspx?action=getsms&token={token}&itemid={itemid}&mobile={phone}&release=1×tamp={time}'
target_url = node_api.format(token=self.token, itemid=self.project_code, phone=phone, time=time.time())
async with session.get(target_url) as response:
message = await response.text()
rest = self.filter_info(message)
if rest:
return phone + rest
await self.join_black_list(session, phone) # 获取失败加入黑名单
return '%s 短信获取失败' % phone
async def fetch_phone(self, session):
"""获取手机
"""
get_phone_api = 'http://api.fxhyd.cn/UserInterface.aspx?action=getmobile&token={token}&itemid={itemid}×tamp={time}'
target_url = get_phone_api.format(token=self.token, itemid=self.project_code, time=time.time())
async with session.get(target_url) as response:
message = await response.text()
return self.filter_info(message)
async def work(self):
"""开始工作
"""
async with aiohttp.ClientSession() as session:
result = await self.fetch_phone(session)
print('提取手机', result)
# 短信接收
result2 = await self.get_node(session, result)
print(result2)
# 释放
result3 = await self.del_phone(session, result)
print(result3)
async def del_phone(self, session, phone):
"""默认收到短信自动释放手机号，特殊情况可以单独调用释放
"""
del_phone_api = 'http://api.fxhyd.cn/UserInterface.aspx?action=release&token={token}&itemid={itemid}&mobile={phone}'
target_url = del_phone_api.format(token=self.token, itemid=self.project_code, phone=phone)
async with session.get(target_url) as response:
message = await response.text()
return self.filter_info(message)
async def join_black_list(self, session, phone):
"""拉入黑名单，获取短信失败情况下拉入黑名单
"""
black_list_api = 'http://api.fxhyd.cn/UserInterface.aspx?action=addignore&token={token}&itemid={itemid}&mobile={phone}'
target_url = black_list_api.format(token=self.token, itemid=self.project_code, phone=phone)
async with session.get(target_url) as response:
message = await response.text()
return self.filter_info(message)
def start(self):
"""开始工作
"""
loop = asyncio.get_event_loop()
tasks = [asyncio.ensure_future(self.work()) for _ in range(self.aio_count)]
loop.run_until_complete(asyncio.wait(tasks))
if __name__ == '__main__':
"""验证码平台 http://www.51ym.me/User/Default.aspx"""
ym = YMReg('后台查询token', '填写int类型项目编号，后台查询', 1)
ym.start()
================================================
FILE: 原创爬虫工具/Register/README.md
================================================
# 注册短信并发异步接收器
> 基于短信接收平台的异步短信接收器，最大并发上限20，Python3.5+
`pip3 install asyncio`
`pip3 install aiohttp`
[平台网址](http://www.51ym.me/User/Default.aspx)
## 使用步骤
1. 实例化对象时填入平台 token
2. 实例化对象时填入后台查询的项目 id
3. 实例化对象时填入手机短信并发上限（最大20并发）
> 启动后会根据设置的异步并发数进行获取手机号码并监听短信接收情况（60秒） 超过60秒后会将未收到短信的手机号拉入黑名单，并是释放。
若要配置具体某个网站使用，还需开发对应的账号注册器，配合调用本短信接收器来达到自动注册账号的功能
================================================
FILE: 原创爬虫工具/Register/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-05-13 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/README.md
================================================
# Zok 组件使用说明
> by: 362416272@qq.com 自用
### 目录
- repetition 内容更新处理
- save 通用持久化存储组件
- random_UA 随机UA
- proxies 阿布云代理组件
**mysql储存**
1. 必须在zok_config中配置要持久化的数据库账户密码
2. 在爬虫项目文件pipelines管道中，引入并使用
```python
from zok.save.to_mysql import SaveToMysqlBase
class CityLandmarkListPipeline(SaveToMysqlBase):
member = 'city' # redis集合名 如果是分布式无需设置
@staticmethod
def get_sql(item):
sql = """INSERT INTO base_city_landmark(city, county, landmark) VALUES ("{city}","{county}","{landmark}") """.format(
city=item['city'],
county=item['county'],
landmark=item['landmark'],
)
return sql
'''必须调用 def_sql(item)方法，并返回sql语句即可'''
```
**随机UA**
```python
# setting.py中 加入即可
DOWNLOADER_MIDDLEWARES = {
'zok.random_UA.ua_random.RandomUserAgentMiddleware': 20,
}
```
**代理ip设置**
```python
# 在setting中配置即可
DOWNLOADER_MIDDLEWARES = {
'zok.proxies.proxies.ProxyMiddleware': 15, # 自定义的中间件
}
```
**基于redis内容去重更新**
> 原理： 在储存数据之前取到hash数据值，并加以对比，如果有值就跳过不储存，无值就set(md5, id)
1. 开启redis服务
2. 在 zok_config中配置 redis配置
3. 应用储存组件 mysql 就会自动启用去重增量更新功能
================================================
FILE: 原创爬虫工具/zok/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/5 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/get_db/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/16 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/get_db/from_mongodb.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-04-23 Python: 3.7
from pymongo import MongoClient
from zok.zok_config import MONGODB_URL
client = MongoClient(MONGODB_URL)
database = client.meituan_db # 链接数据库
collection = database.href_coolections # 链接结合
data = collection.find({},{'_id': 0})
================================================
FILE: 原创爬虫工具/zok/get_db/from_mysql.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
import pymysql
from zok.zok_config import *
def get_data(sql):
conn = pymysql.Connect(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB_NAME,
)
# 创建游标对象
cursor = conn.cursor()
# 提交事务
try:
cursor.execute(sql)
data = cursor.fetchall()
cursor.close()
conn.close()
return data
except Exception as e:
print(e)
print('异常回滚')
conn.rollback()
cursor.close()
conn.close()
return None
================================================
FILE: 原创爬虫工具/zok/proxies/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/proxies/proxies.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/5 Python: 3.7
import base64
from zok.zok_config import *
# 代理服务器
proxyServer = "http://http-dyn.abuyun.com:9020"
proxyAuth = "Basic " + base64.urlsafe_b64encode(bytes((PROXY_USER + ":" + PROXY_PASS), "ascii")).decode("utf8")
class ProxyMiddleware(object):
"""自定义中间件代理IP"""
def process_request(self, request, spider):
request.meta["proxy"] = proxyServer
request.headers["Proxy-Authorization"] = proxyAuth
================================================
FILE: 原创爬虫工具/zok/random_UA/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/random_UA/fake_useragent.json
================================================
{
"browsers": {
"chrome": ["Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14"],
"opera": ["Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"],
"firefox": ["Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0", "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0", "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0", "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0", "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0", "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6"],
"internetexplorer": ["Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)"],
"safari": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10", "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5"]
},
"randomize": {
"344": "chrome",
"819": "firefox",
"346": "chrome",
"347": "chrome",
"340": "chrome",
"341": "chrome",
"342": "chrome",
"343": "chrome",
"810": "internetexplorer",
"811": "internetexplorer",
"812": "internetexplorer",
"813": "firefox",
"348": "chrome",
"349": "chrome",
"816": "firefox",
"817": "firefox",
"737": "chrome",
"719": "chrome",
"718": "chrome",
"717": "chrome",
"716": "chrome",
"715": "chrome",
"714": "chrome",
"713": "chrome",
"712": "chrome",
"711": "chrome",
"710": "chrome",
"421": "chrome",
"129": "chrome",
"420": "chrome",
"423": "chrome",
"422": "chrome",
"425": "chrome",
"619": "chrome",
"424": "chrome",
"427": "chrome",
"298": "chrome",
"299": "chrome",
"296": "chrome",
"297": "chrome",
"294": "chrome",
"295": "chrome",
"292": "chrome",
"293": "chrome",
"290": "chrome",
"291": "chrome",
"591": "chrome",
"590": "chrome",
"593": "chrome",
"592": "chrome",
"595": "chrome",
"594": "chrome",
"597": "chrome",
"596": "chrome",
"195": "chrome",
"194": "chrome",
"197": "chrome",
"196": "chrome",
"191": "chrome",
"190": "chrome",
"193": "chrome",
"192": "chrome",
"270": "chrome",
"271": "chrome",
"272": "chrome",
"273": "chrome",
"274": "chrome",
"275": "chrome",
"276": "chrome",
"277": "chrome",
"278": "chrome",
"279": "chrome",
"569": "chrome",
"497": "chrome",
"524": "chrome",
"525": "chrome",
"526": "chrome",
"527": "chrome",
"520": "chrome",
"521": "chrome",
"522": "chrome",
"523": "chrome",
"528": "chrome",
"529": "chrome",
"449": "chrome",
"448": "chrome",
"345": "chrome",
"443": "chrome",
"442": "chrome",
"441": "chrome",
"440": "chrome",
"447": "chrome",
"446": "chrome",
"445": "chrome",
"444": "chrome",
"47": "chrome",
"108": "chrome",
"109": "chrome",
"102": "chrome",
"103": "chrome",
"100": "chrome",
"101": "chrome",
"106": "chrome",
"107": "chrome",
"104": "chrome",
"105": "chrome",
"902": "firefox",
"903": "firefox",
"39": "chrome",
"38": "chrome",
"906": "firefox",
"907": "firefox",
"904": "firefox",
"905": "firefox",
"33": "chrome",
"32": "chrome",
"31": "chrome",
"30": "chrome",
"37": "chrome",
"36": "chrome",
"35": "chrome",
"34": "chrome",
"641": "chrome",
"640": "chrome",
"643": "chrome",
"642": "chrome",
"645": "chrome",
"644": "chrome",
"438": "chrome",
"439": "chrome",
"436": "chrome",
"437": "chrome",
"434": "chrome",
"435": "chrome",
"432": "chrome",
"433": "chrome",
"430": "chrome",
"431": "chrome",
"826": "firefox",
"339": "chrome",
"338": "chrome",
"335": "chrome",
"334": "chrome",
"337": "chrome",
"336": "chrome",
"331": "chrome",
"330": "chrome",
"333": "chrome",
"332": "chrome",
"559": "chrome",
"745": "chrome",
"854": "firefox",
"818": "firefox",
"856": "firefox",
"857": "firefox",
"850": "firefox",
"851": "firefox",
"852": "firefox",
"0": "chrome",
"858": "firefox",
"859": "firefox",
"748": "chrome",
"6": "chrome",
"43": "chrome",
"99": "chrome",
"98": "chrome",
"91": "chrome",
"90": "chrome",
"93": "chrome",
"92": "chrome",
"95": "chrome",
"94": "chrome",
"97": "chrome",
"96": "chrome",
"814": "firefox",
"815": "firefox",
"153": "chrome",
"740": "chrome",
"741": "chrome",
"742": "chrome",
"743": "chrome",
"744": "chrome",
"558": "chrome",
"746": "chrome",
"747": "chrome",
"555": "chrome",
"554": "chrome",
"557": "chrome",
"556": "chrome",
"551": "chrome",
"550": "chrome",
"553": "chrome",
"552": "chrome",
"238": "chrome",
"239": "chrome",
"234": "chrome",
"235": "chrome",
"236": "chrome",
"237": "chrome",
"230": "chrome",
"231": "chrome",
"232": "chrome",
"233": "chrome",
"1": "chrome",
"155": "chrome",
"146": "chrome",
"147": "chrome",
"618": "chrome",
"145": "chrome",
"142": "chrome",
"143": "chrome",
"140": "chrome",
"141": "chrome",
"612": "chrome",
"613": "chrome",
"610": "chrome",
"611": "chrome",
"616": "chrome",
"617": "chrome",
"148": "chrome",
"149": "chrome",
"46": "chrome",
"154": "chrome",
"948": "safari",
"949": "safari",
"946": "safari",
"947": "safari",
"944": "safari",
"945": "safari",
"942": "safari",
"943": "safari",
"940": "safari",
"941": "safari",
"689": "chrome",
"688": "chrome",
"685": "chrome",
"684": "chrome",
"687": "chrome",
"686": "chrome",
"681": "chrome",
"680": "chrome",
"683": "chrome",
"682": "chrome",
"458": "chrome",
"459": "chrome",
"133": "chrome",
"132": "chrome",
"131": "chrome",
"130": "chrome",
"137": "chrome",
"136": "chrome",
"135": "chrome",
"134": "chrome",
"494": "chrome",
"495": "chrome",
"139": "chrome",
"138": "chrome",
"490": "chrome",
"491": "chrome",
"492": "chrome",
"493": "chrome",
"24": "chrome",
"25": "chrome",
"26": "chrome",
"27": "chrome",
"20": "chrome",
"21": "chrome",
"22": "chrome",
"23": "chrome",
"28": "chrome",
"29": "chrome",
"820": "firefox",
"407": "chrome",
"406": "chrome",
"405": "chrome",
"404": "chrome",
"403": "chrome",
"402": "chrome",
"401": "chrome",
"400": "chrome",
"933": "firefox",
"932": "firefox",
"931": "firefox",
"930": "firefox",
"937": "safari",
"452": "chrome",
"409": "chrome",
"408": "chrome",
"453": "chrome",
"414": "chrome",
"183": "chrome",
"415": "chrome",
"379": "chrome",
"378": "chrome",
"228": "chrome",
"829": "firefox",
"828": "firefox",
"371": "chrome",
"370": "chrome",
"373": "chrome",
"372": "chrome",
"375": "chrome",
"374": "chrome",
"377": "chrome",
"376": "chrome",
"708": "chrome",
"709": "chrome",
"704": "chrome",
"705": "chrome",
"706": "chrome",
"707": "chrome",
"700": "chrome",
"144": "chrome",
"702": "chrome",
"703": "chrome",
"393": "chrome",
"392": "chrome",
"88": "chrome",
"89": "chrome",
"397": "chrome",
"396": "chrome",
"395": "chrome",
"394": "chrome",
"82": "chrome",
"83": "chrome",
"80": "chrome",
"81": "chrome",
"86": "chrome",
"87": "chrome",
"84": "chrome",
"85": "chrome",
"797": "internetexplorer",
"796": "internetexplorer",
"795": "internetexplorer",
"794": "internetexplorer",
"793": "internetexplorer",
"792": "internetexplorer",
"791": "internetexplorer",
"790": "internetexplorer",
"749": "chrome",
"799": "internetexplorer",
"798": "internetexplorer",
"7": "chrome",
"170": "chrome",
"586": "chrome",
"587": "chrome",
"584": "chrome",
"585": "chrome",
"582": "chrome",
"583": "chrome",
"580": "chrome",
"581": "chrome",
"588": "chrome",
"589": "chrome",
"245": "chrome",
"244": "chrome",
"247": "chrome",
"246": "chrome",
"241": "chrome",
"614": "chrome",
"243": "chrome",
"242": "chrome",
"615": "chrome",
"249": "chrome",
"248": "chrome",
"418": "chrome",
"419": "chrome",
"519": "chrome",
"518": "chrome",
"511": "chrome",
"510": "chrome",
"513": "chrome",
"512": "chrome",
"515": "chrome",
"514": "chrome",
"517": "chrome",
"516": "chrome",
"623": "chrome",
"622": "chrome",
"621": "chrome",
"620": "chrome",
"627": "chrome",
"626": "chrome",
"625": "chrome",
"624": "chrome",
"450": "chrome",
"451": "chrome",
"629": "chrome",
"628": "chrome",
"454": "chrome",
"455": "chrome",
"456": "chrome",
"457": "chrome",
"179": "chrome",
"178": "chrome",
"177": "chrome",
"199": "chrome",
"175": "chrome",
"174": "chrome",
"173": "chrome",
"172": "chrome",
"171": "chrome",
"198": "chrome",
"977": "opera",
"182": "chrome",
"975": "opera",
"974": "opera",
"973": "opera",
"972": "opera",
"971": "opera",
"970": "opera",
"180": "chrome",
"979": "opera",
"978": "opera",
"656": "chrome",
"599": "chrome",
"654": "chrome",
"181": "chrome",
"186": "chrome",
"187": "chrome",
"184": "chrome",
"185": "chrome",
"652": "chrome",
"188": "chrome",
"189": "chrome",
"658": "chrome",
"653": "chrome",
"650": "chrome",
"651": "chrome",
"11": "chrome",
"10": "chrome",
"13": "chrome",
"12": "chrome",
"15": "chrome",
"14": "chrome",
"17": "chrome",
"16": "chrome",
"19": "chrome",
"18": "chrome",
"863": "firefox",
"862": "firefox",
"865": "firefox",
"864": "firefox",
"867": "firefox",
"866": "firefox",
"354": "chrome",
"659": "chrome",
"44": "chrome",
"883": "firefox",
"882": "firefox",
"881": "firefox",
"880": "firefox",
"887": "firefox",
"886": "firefox",
"885": "firefox",
"884": "firefox",
"889": "firefox",
"888": "firefox",
"116": "chrome",
"45": "chrome",
"657": "chrome",
"355": "chrome",
"322": "chrome",
"323": "chrome",
"320": "chrome",
"321": "chrome",
"326": "chrome",
"327": "chrome",
"324": "chrome",
"325": "chrome",
"328": "chrome",
"329": "chrome",
"562": "chrome",
"201": "chrome",
"200": "chrome",
"203": "chrome",
"202": "chrome",
"205": "chrome",
"204": "chrome",
"207": "chrome",
"206": "chrome",
"209": "chrome",
"208": "chrome",
"779": "internetexplorer",
"778": "internetexplorer",
"77": "chrome",
"76": "chrome",
"75": "chrome",
"74": "chrome",
"73": "chrome",
"72": "chrome",
"71": "chrome",
"70": "chrome",
"655": "chrome",
"567": "chrome",
"79": "chrome",
"78": "chrome",
"359": "chrome",
"358": "chrome",
"669": "chrome",
"668": "chrome",
"667": "chrome",
"666": "chrome",
"665": "chrome",
"664": "chrome",
"663": "chrome",
"662": "chrome",
"661": "chrome",
"660": "chrome",
"215": "chrome",
"692": "chrome",
"693": "chrome",
"690": "chrome",
"691": "chrome",
"696": "chrome",
"697": "chrome",
"694": "chrome",
"695": "chrome",
"698": "chrome",
"699": "chrome",
"542": "chrome",
"543": "chrome",
"540": "chrome",
"541": "chrome",
"546": "chrome",
"547": "chrome",
"544": "chrome",
"545": "chrome",
"8": "chrome",
"548": "chrome",
"549": "chrome",
"598": "chrome",
"869": "firefox",
"868": "firefox",
"120": "chrome",
"121": "chrome",
"122": "chrome",
"123": "chrome",
"124": "chrome",
"125": "chrome",
"126": "chrome",
"127": "chrome",
"128": "chrome",
"2": "chrome",
"219": "chrome",
"176": "chrome",
"214": "chrome",
"563": "chrome",
"928": "firefox",
"929": "firefox",
"416": "chrome",
"417": "chrome",
"410": "chrome",
"411": "chrome",
"412": "chrome",
"413": "chrome",
"920": "firefox",
"498": "chrome",
"922": "firefox",
"923": "firefox",
"924": "firefox",
"925": "firefox",
"926": "firefox",
"927": "firefox",
"319": "chrome",
"318": "chrome",
"313": "chrome",
"312": "chrome",
"311": "chrome",
"310": "chrome",
"317": "chrome",
"316": "chrome",
"315": "chrome",
"314": "chrome",
"921": "firefox",
"496": "chrome",
"832": "firefox",
"833": "firefox",
"830": "firefox",
"831": "firefox",
"836": "firefox",
"837": "firefox",
"834": "firefox",
"835": "firefox",
"838": "firefox",
"839": "firefox",
"3": "chrome",
"368": "chrome",
"369": "chrome",
"366": "chrome",
"367": "chrome",
"364": "chrome",
"365": "chrome",
"362": "chrome",
"363": "chrome",
"360": "chrome",
"361": "chrome",
"218": "chrome",
"380": "chrome",
"861": "firefox",
"382": "chrome",
"383": "chrome",
"384": "chrome",
"385": "chrome",
"386": "chrome",
"387": "chrome",
"388": "chrome",
"389": "chrome",
"784": "internetexplorer",
"785": "internetexplorer",
"786": "internetexplorer",
"787": "internetexplorer",
"780": "internetexplorer",
"781": "internetexplorer",
"782": "internetexplorer",
"381": "chrome",
"788": "internetexplorer",
"789": "internetexplorer",
"860": "firefox",
"151": "chrome",
"579": "chrome",
"578": "chrome",
"150": "chrome",
"573": "chrome",
"572": "chrome",
"571": "chrome",
"570": "chrome",
"577": "chrome",
"576": "chrome",
"575": "chrome",
"574": "chrome",
"60": "chrome",
"61": "chrome",
"62": "chrome",
"259": "chrome",
"64": "chrome",
"65": "chrome",
"66": "chrome",
"67": "chrome",
"68": "chrome",
"253": "chrome",
"250": "chrome",
"251": "chrome",
"256": "chrome",
"257": "chrome",
"254": "chrome",
"255": "chrome",
"499": "chrome",
"157": "chrome",
"156": "chrome",
"939": "safari",
"731": "chrome",
"730": "chrome",
"733": "chrome",
"938": "safari",
"735": "chrome",
"734": "chrome",
"508": "chrome",
"736": "chrome",
"506": "chrome",
"738": "chrome",
"504": "chrome",
"505": "chrome",
"502": "chrome",
"503": "chrome",
"500": "chrome",
"501": "chrome",
"630": "chrome",
"631": "chrome",
"632": "chrome",
"633": "chrome",
"469": "chrome",
"468": "chrome",
"636": "chrome",
"637": "chrome",
"465": "chrome",
"464": "chrome",
"467": "chrome",
"466": "chrome",
"461": "chrome",
"900": "firefox",
"463": "chrome",
"462": "chrome",
"901": "firefox",
"168": "chrome",
"169": "chrome",
"164": "chrome",
"165": "chrome",
"166": "chrome",
"167": "chrome",
"160": "chrome",
"161": "chrome",
"162": "chrome",
"163": "chrome",
"964": "safari",
"965": "safari",
"966": "safari",
"967": "safari",
"960": "safari",
"961": "safari",
"962": "safari",
"963": "safari",
"783": "internetexplorer",
"968": "safari",
"969": "opera",
"936": "firefox",
"935": "firefox",
"934": "firefox",
"908": "firefox",
"909": "firefox",
"722": "chrome",
"426": "chrome",
"878": "firefox",
"879": "firefox",
"876": "firefox",
"877": "firefox",
"874": "firefox",
"875": "firefox",
"872": "firefox",
"873": "firefox",
"870": "firefox",
"871": "firefox",
"9": "chrome",
"890": "firefox",
"891": "firefox",
"892": "firefox",
"893": "firefox",
"894": "firefox",
"647": "chrome",
"896": "firefox",
"897": "firefox",
"898": "firefox",
"899": "firefox",
"646": "chrome",
"649": "chrome",
"648": "chrome",
"357": "chrome",
"356": "chrome",
"809": "internetexplorer",
"808": "internetexplorer",
"353": "chrome",
"352": "chrome",
"351": "chrome",
"350": "chrome",
"803": "internetexplorer",
"802": "internetexplorer",
"801": "internetexplorer",
"800": "internetexplorer",
"807": "internetexplorer",
"806": "internetexplorer",
"805": "internetexplorer",
"804": "internetexplorer",
"216": "chrome",
"217": "chrome",
"768": "chrome",
"769": "chrome",
"212": "chrome",
"213": "chrome",
"210": "chrome",
"211": "chrome",
"762": "chrome",
"763": "chrome",
"760": "chrome",
"761": "chrome",
"766": "chrome",
"767": "chrome",
"764": "chrome",
"765": "chrome",
"40": "chrome",
"41": "chrome",
"289": "chrome",
"288": "chrome",
"4": "chrome",
"281": "chrome",
"280": "chrome",
"283": "chrome",
"282": "chrome",
"285": "chrome",
"284": "chrome",
"287": "chrome",
"286": "chrome",
"678": "chrome",
"679": "chrome",
"674": "chrome",
"675": "chrome",
"676": "chrome",
"677": "chrome",
"670": "chrome",
"671": "chrome",
"672": "chrome",
"673": "chrome",
"263": "chrome",
"262": "chrome",
"261": "chrome",
"260": "chrome",
"267": "chrome",
"266": "chrome",
"265": "chrome",
"264": "chrome",
"269": "chrome",
"268": "chrome",
"59": "chrome",
"58": "chrome",
"55": "chrome",
"54": "chrome",
"57": "chrome",
"56": "chrome",
"51": "chrome",
"258": "chrome",
"53": "chrome",
"52": "chrome",
"537": "chrome",
"536": "chrome",
"535": "chrome",
"63": "chrome",
"533": "chrome",
"532": "chrome",
"531": "chrome",
"530": "chrome",
"152": "chrome",
"539": "chrome",
"538": "chrome",
"775": "internetexplorer",
"774": "internetexplorer",
"982": "opera",
"983": "opera",
"980": "opera",
"981": "opera",
"777": "internetexplorer",
"984": "opera",
"50": "chrome",
"115": "chrome",
"252": "chrome",
"117": "chrome",
"776": "internetexplorer",
"111": "chrome",
"110": "chrome",
"113": "chrome",
"69": "chrome",
"771": "chrome",
"119": "chrome",
"118": "chrome",
"770": "chrome",
"773": "internetexplorer",
"772": "internetexplorer",
"429": "chrome",
"428": "chrome",
"534": "chrome",
"919": "firefox",
"918": "firefox",
"915": "firefox",
"914": "firefox",
"917": "firefox",
"916": "firefox",
"911": "firefox",
"910": "firefox",
"913": "firefox",
"912": "firefox",
"308": "chrome",
"309": "chrome",
"855": "firefox",
"300": "chrome",
"301": "chrome",
"302": "chrome",
"303": "chrome",
"304": "chrome",
"305": "chrome",
"306": "chrome",
"307": "chrome",
"895": "firefox",
"825": "firefox",
"824": "firefox",
"827": "firefox",
"847": "firefox",
"846": "firefox",
"845": "firefox",
"844": "firefox",
"843": "firefox",
"842": "firefox",
"841": "firefox",
"840": "firefox",
"821": "firefox",
"853": "firefox",
"849": "firefox",
"848": "firefox",
"823": "firefox",
"822": "firefox",
"240": "chrome",
"390": "chrome",
"732": "chrome",
"753": "chrome",
"752": "chrome",
"751": "chrome",
"750": "chrome",
"757": "chrome",
"756": "chrome",
"755": "chrome",
"754": "chrome",
"560": "chrome",
"561": "chrome",
"759": "chrome",
"758": "chrome",
"564": "chrome",
"565": "chrome",
"566": "chrome",
"701": "chrome",
"739": "chrome",
"229": "chrome",
"507": "chrome",
"227": "chrome",
"226": "chrome",
"225": "chrome",
"224": "chrome",
"223": "chrome",
"222": "chrome",
"221": "chrome",
"220": "chrome",
"114": "chrome",
"391": "chrome",
"726": "chrome",
"727": "chrome",
"724": "chrome",
"725": "chrome",
"568": "chrome",
"723": "chrome",
"720": "chrome",
"721": "chrome",
"728": "chrome",
"729": "chrome",
"605": "chrome",
"604": "chrome",
"607": "chrome",
"606": "chrome",
"601": "chrome",
"600": "chrome",
"603": "chrome",
"602": "chrome",
"159": "chrome",
"158": "chrome",
"112": "chrome",
"609": "chrome",
"608": "chrome",
"976": "opera",
"634": "chrome",
"399": "chrome",
"635": "chrome",
"959": "safari",
"958": "safari",
"398": "chrome",
"48": "chrome",
"49": "chrome",
"951": "safari",
"950": "safari",
"953": "safari",
"952": "safari",
"42": "chrome",
"954": "safari",
"957": "safari",
"956": "safari",
"638": "chrome",
"5": "chrome",
"639": "chrome",
"460": "chrome",
"489": "chrome",
"488": "chrome",
"487": "chrome",
"486": "chrome",
"485": "chrome",
"484": "chrome",
"483": "chrome",
"482": "chrome",
"481": "chrome",
"480": "chrome",
"509": "chrome",
"955": "safari",
"472": "chrome",
"473": "chrome",
"470": "chrome",
"471": "chrome",
"476": "chrome",
"477": "chrome",
"474": "chrome",
"475": "chrome",
"478": "chrome",
"479": "chrome"
}
}
================================================
FILE: 原创爬虫工具/zok/random_UA/ua_random.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
import os
from fake_useragent import UserAgent
class RandomUserAgentMiddleware(object):
"""
first to use location because it is the fastest
"""
def __init__(self):
location = os.getcwd() + '/zok/random_UA/fake_useragent.json'
self.agent = UserAgent(path=location) # 调用本地 ua池
# self.agent = UserAgent(verify_ssl=False)
# self.agent = UserAgent(use_cache_server=False)
@classmethod
def from_crawler(cls, crawler):
return cls()
def process_request(self, request, spider):
request.headers.setdefault('User-Agent', self.agent.random)
================================================
FILE: 原创爬虫工具/zok/repetition/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/5 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/repetition/update_cache.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
import redis
import hashlib
from zok.zok_config import REDIS_PORT, REDIS_DB_NAME, REDIS_HOST, REDIS_USER, REDIS_PASSWORD
class CacheRedis(object):
pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB_NAME, password=REDIS_PASSWORD, decode_responses=True)
r = redis.Redis(connection_pool=pool)
# 加上decode_responses=True，写入的键值对中的value为str类型，不加这个参数写入的则为字节类型。
# 1. 根据储存数据取值判断是否存在
# 3. 不存在-已有数据: 需要更新
# 4. 不存在-无数据: 需要插入
# 5. 存在 直接跳过储存
# BUG 在redis数据库丢失的情况下【会全体重新录入】
@staticmethod
def get_md5(data):
md5 = hashlib.md5(data.encode('utf-8')).hexdigest()
return md5
def redis_exists(self, member, md5):
"""
验证hash是否存在， 有返回True，没有返回False
:param member: 验证区域集合Key
:param md5: 要储存的数据
:return: True or False
"""
print()
if self.r.sismember(member, md5):
return True
else:
return False
def save_redis(self, member, md5):
self.r.sadd(member, md5)
================================================
FILE: 原创爬虫工具/zok/save/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
================================================
FILE: 原创爬虫工具/zok/save/to_mysql.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
import pymysql
from zok.zok_config import *
from zok.repetition.update_cache import CacheRedis
class SaveToMysqlBase(object):
"""
mysql储存基类
新增语法 INSERT INTO 表名(city, county, district) VALUES ("%s","%s","%s")
更新语法 UPDATE 表名 SET mail = "playstation.com" WHERE user_name = "Peter"
"""
member = None # 不设置默认不开启 redis去重校验
conn = None
cursor = None # 游标对象
redis = CacheRedis()
def open_spider(self, spider):
print('开始爬虫，链接数据库')
self.conn = pymysql.Connect(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB_NAME,
)
def process_item(self, item, spider):
# 写sql语句 插数据，没有表的话要先在数据库创建
sql = self.get_sql(item)
if self.member:
sql_md5 = self.redis.get_md5(sql)
if not self.redis.redis_exists(self.member, sql_md5):
# 创建游标对象
self.cursor = self.conn.cursor()
# 提交事务
try:
self.cursor.execute(sql)
self.conn.commit()
self.redis.save_redis(self.member, sql_md5)
# int(conn.insert_id()) # 最新插入行的主键ID，conn.insert_id()一定要在conn.commit()之前，否则会返回0
except Exception as e:
print(e)
print('异常回滚')
self.conn.rollback()
self.cursor.close()
return item
else:
print('已有相同数据无需插入')
else:
# 创建游标对象
self.cursor = self.conn.cursor()
# 提交事务
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
print(e)
print('异常回滚')
self.conn.rollback()
self.cursor.close()
return item
def close_spider(self, spider):
print('爬虫结束, 关闭通道')
self.conn.close()
================================================
FILE: 原创爬虫工具/zok/zok_config.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/5 Python: 3.7
from urllib import parse
MONGODB_URL = 'mongodb://localhost:27017'
REDIS_HOST = "localhost"
REDIS_USER = "root"
REDIS_PASSWORD = ""
REDIS_DB_NAME = 0
REDIS_PORT = 6379
================================================
FILE: 滑动验证码/【w3c】滑块验证/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-10 Python: 3.7
================================================
FILE: 滑动验证码/【w3c】滑块验证/w3c.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-10 Python: 3.7
import requests
import re
import json
from PIL import Image # 图像处理模块
from io import BytesIO # io 加载
class W3C:
"""
滑块验证码识别与破解
w3c 网站滑块网址 https://www.w3cschool.cn/register
"""
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
'Host': 'www.w3cschool.cn'
}
self.local = 'https://www.w3cschool.cn'
self.check = 'https://www.w3cschool.cn/dragcheck'
self.session = requests.session()
self.height = 58 # 碎片高
self.width = 13 # 碎片宽
def get_img(self):
"""获取页面滑块图片"""
response = self.session.get('https://www.w3cschool.cn/register', headers=self.headers)
results = re.findall(r'background-image: url\("(.*?)"\);', response.text)
bg_img = self.local + results[0] # 背景图
hk_img = self.local + results[1] # 滑块图
# print(bg_img, hk_img)
# 取 css
css_list = re.findall(r'background-position:(.*?);">', response.text)
# 缺口滑块
ret = self.session.get(hk_img)
with open('hk.png', 'wb') as f:
f.write(ret.content)
# 图片对象
img_cha = self.split_img(bg_img, css_list)
xy = None # 识别结果
for n in range(4):
full_image = Image.open('img/'+str(n)+'.png')
# 获取缺口位置
xy = self.get_distance(img_cha, full_image)
if xy and xy > 0:
break
return xy
def get_distance(self, bg_image, full_image):
"""
计算滑块移动距离
:param bg_image: (Image) 有缺口的整图
:param full_image: (Image) 完整原图
:return: (Int)缺口离滑块的距离
"""
# 滑块的初始位置
distance = 0
# 遍历像素点横坐标
for i in range(distance, full_image.size[0]):
# 遍历像素点纵坐标
for j in range(full_image.size[1]):
# 如果不是相同像素
if not self.is_pixel_equal(bg_image, full_image, i, j):
# 返回此时横轴坐标就是滑块需要移动的距离
return i
@staticmethod
def is_pixel_equal(bg_image, full_image, x, y):
"""
判断像素是否相同算法
:param bg_image: (Image)缺口图片
:param full_image: (Image)完整图片
:param x: (Int)位置x
:param y: (Int)位置y
:return: (Boolean)像素是否相同
"""
# 获取缺口图片的像素点(按照RGB格式)
bg_pixel = bg_image.load()[x, y]
# 获取完整图片的像素点(按照RGB格式)
full_pixel = full_image.load()[x, y]
# 设置一个判定值，像素值之差超过判定值则认为该像素不相同
threshold = 1 # 可根据识别效果调整
# 判断像素的各个颜色之差，abs()用于取绝对值
if (abs(bg_pixel[0] - full_pixel[0] < threshold) and abs(bg_pixel[1] - full_pixel[1] < threshold) and abs(
bg_pixel[2] - full_pixel[2] < threshold)):
# 如果差值在判断值之内，返回是相同像素
return True
else:
# 如果差值在判断值之外，返回不是相同像素
return False
def split_img(self, bg_img, css_list):
"""图片还原"""
# 背景
ret = self.session.get(bg_img)
bytes_io = BytesIO()
bytes_io.write(ret.content)
img = Image.open(bytes_io)
img_cha = Image.open('chache.png')
img.save('bg.png') # 记录一下原始图片
for index, i in enumerate(css_list):
x, y = [abs(int(xy[:-2])) for xy in i.split(' ')] # 将 css 坐标抽离出来
box = [x, y, x + self.width, y + self.height]
im_crop = img.crop(box)
if index < 20: # 第一行
img_cha.paste(im_crop, (index*self.width, 0))
else: # 第二行
img_cha.paste(im_crop, ((index-20)*self.width, self.height))
img_cha.save('chache.png') # 储存
return img_cha
def check_code(self):
"""滑块认证"""
point = self.get_img()
if point:
print('\033[1;34m识别成功: 缺口位置 x: %s \033[0m' % point)
print('\033[1;36m提交数据中...\033[0m')
result = self.session.post(self.check, data={'point': point}, headers=self.headers)
print(json.loads(result.text).get('message'))
else:
print('滑块识别失败。。运气不佳。。重试即可。')
if __name__ == '__main__':
wc = W3C()
wc.check_code()
================================================
FILE: 滑动验证码/【腾讯】滑块验证/discriminate.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-11 Python: 3.7
"""
pip3 install opencv-python
"""
import cv2 as cv
def get_pos(image):
"""
缺口轮廓检测
对付腾讯滑块够用
该方法识别率 95% 左右
"""
blurred = cv.GaussianBlur(image, (5, 5), 0)
canny = cv.Canny(blurred, 200, 400)
contours, hierarchy = cv.findContours(canny, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
for i, contour in enumerate(contours):
m = cv.moments(contour)
if m['m00'] == 0:
cx = cy = 0
else:
cx, cy = m['m10'] / m['m00'], m['m01'] / m['m00']
if 6000 < cv.contourArea(contour) < 8000 and 370 < cv.arcLength(contour, True) < 390:
if cx < 400:
continue
x, y, w, h = cv.boundingRect(contour) # 外接矩形
cv.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)
cv.imshow('image', image)
return x
return 0
if __name__ == '__main__':
"""
这里是滑块缺口识别
识别到后
1。可以通过自动化工具取拖动滑块
2。可以通过参数解析的形式生成参数提交通过验证
"""
img0 = cv.imread('bg.jpeg')
get_pos(img0)
cv.waitKey(0)
cv.destroyAllWindows()
================================================
FILE: 滑动验证码/【腾讯】滑块验证/sel.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-10-28 Python: 3.7
import requests
import time
import cv2 as cv
import random
from selenium.webdriver import ActionChains
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC # 显性等待
from selenium.webdriver.common.by import By
class TX:
def __init__(self):
# 初始化
profile = webdriver.FirefoxOptions() # 配置无头
profile.add_argument('-headless')
self.browser = webdriver.Firefox()
wait = WebDriverWait(self.browser, 10)
self.api_url = ''
def end(self):
self.browser.quit()
def tx_code(self):
"""这里需要访问，带有滑动验证码的页面，然后会获取滑块对其进行滑动"""
WebDriverWait(self.browser, 20, 0.5).until(EC.presence_of_element_located((By.ID, 'tcaptcha_iframe'))) # 等待 iframe
self.browser.switch_to.frame(self.browser.find_element_by_id('tcaptcha_iframe')) # 加载 iframe
time.sleep(0.5)
bk_block = self.browser.find_element_by_xpath('//img[@id="slideBg"]').get_attribute('src')
print(bk_block)
if self.save_img(bk_block):
dex = self.get_pos()
track_list = self.get_track(dex)
time.sleep(0.5)
slid_ing = self.browser.find_element_by_xpath('//div[@id="tcaptcha_drag_thumb"]') # 滑块定位
ActionChains(self.browser).click_and_hold(on_element=slid_ing).perform() # 鼠标按下
time.sleep(0.2)
print('轨迹', track_list)
for track in track_list:
ActionChains(self.browser).move_by_offset(xoffset=track, yoffset=0).perform() # 鼠标移动到距离当前位置（x,y）
time.sleep(1)
ActionChains(self.browser).release(on_element=slid_ing).perform() # print('第三步,释放鼠标')
time.sleep(1)
# 识别图片
return True
else:
print('缺口图片捕获失败')
return False
@staticmethod
def save_img(bk_block):
"""保存图片"""
try:
img = requests.get(bk_block).content
with open('bg.jpeg', 'wb') as f:
f.write(img)
return True
except:
return False
@staticmethod
def get_pos():
"""识别缺口
注意：网页上显示的图片为缩放图片，缩放 50% 所以识别坐标需要 0.5
"""
image = cv.imread('bg.jpeg')
blurred = cv.GaussianBlur(image, (5, 5), 0)
canny = cv.Canny(blurred, 200, 400)
contours, hierarchy = cv.findContours(canny, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
for i, contour in enumerate(contours):
m = cv.moments(contour)
if m['m00'] == 0:
cx = cy = 0
else:
cx, cy = m['m10'] / m['m00'], m['m01'] / m['m00']
if 6000 < cv.contourArea(contour) < 8000 and 370 < cv.arcLength(contour, True) < 390:
if cx < 400:
continue
x, y, w, h = cv.boundingRect(contour) # 外接矩形
cv.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)
# cv.imshow('image', image) # 显示识别结果
print('【缺口识别】 {x}px'.format(x=x/2))
return x/2
return 0
@staticmethod
def get_track(distance):
"""模拟轨迹
"""
distance -= 37 # 初始位置
# 初速度
v = 0
# 单位时间为0.2s来统计轨迹，轨迹即0.2内的位移
t = 0.2
# 位移/轨迹列表，列表内的一个元素代表0.2s的位移
tracks = []
# 当前的位移
current = 0
# 到达mid值开始减速
mid = distance * 7 / 8
distance += 10 # 先滑过一点，最后再反着滑动回来
# a = random.randint(1,3)
while current < distance:
if current < mid:
# 加速度越小，单位时间的位移越小,模拟的轨迹就越多越详细
a = random.randint(2, 4) # 加速运动
else:
a = -random.randint(3, 5) # 减速运动
# 初速度
v0 = v
# 0.2秒时间内的位移
s = v0 * t + 0.5 * a * (t ** 2)
# 当前的位置
current += s
# 添加到轨迹列表
tracks.append(round(s))
# 速度已经达到v,该速度作为下次的初速度
v = v0 + a * t
# 反着滑动到大概准确位置
for i in range(4):
tracks.append(-random.randint(2, 3))
for i in range(4):
tracks.append(-random.randint(1, 3))
return tracks
def move_to(self, index):
"""滑动滑块"""
pass
def re_start(self):
self.tx_code()
self.end()
if __name__ == '__main__':
"""
请在 tx_code 中访问含有 腾讯滑块的页面，然后会对其自动滑动
"""
xbt = TX()
xbt.re_start()
================================================
FILE: 项目/HouseScrapy/requirements
================================================
scrapy
scrapy-redis
pymysql
redis>=3.2.1
pymongo
================================================
FILE: 项目/HouseScrapy/scrapy.cfg
================================================
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = settings
[deploy]
#url = http://localhost:6800/
project = HouseScrapy
================================================
FILE: 项目/HouseScrapy/settings.py
================================================
# -*- coding: utf-8 -*-
# 阿布云代理 IP
PROXY_USER = ''
PROXY_PASS = ''
BOT_NAME = 'HouseScrapy'
SPIDER_MODULES = ['spiders']
NEWSPIDER_MODULE = 'spiders'
# 否认协议
ROBOTSTXT_OBEY = False
# 随机延迟
RANDOMIZE_DOWNLOAD_DELAY = True
# 重试处理
DOWNLOAD_FAIL_ON_DATALOSS = False
# 设置超时时间
DOWNLOAD_TIMEOUT = 5
# MongoDB
MONGODB_URL = 'mongodb://localhost:27017'
MONGODB_DB = '房产'
MONGODB_COLL = '地产数据'
# Redis
REDIS_HOST = '127.0.0.1' # 本机
REDIS_WORD = None
REDIS_PORT = 6379
# 限流 秒/次
DOWNLOAD_DELAY = 1 / 10
# 禁止301
# HTTPERROR_ALLOWED_CODES = [301]
# 日志配置
# LOG_LEVEL = 'WARNING'
# LOG_FILE = 'log/error_log.txt'
# Headers
DEFAULT_REQUEST_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
'Host': 'www.funi.com'
}
"""项目独立配置区"""
# HOST
HOST = 'http://www.funi.com'
"""===== 分布式配置区 ====="""
# # 去重，利用set指纹去重
# DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'
#
# # 调度器
# SCHEDULER = 'scrapy_redis.scheduler.Scheduler'
#
# # 去重指纹的set
# SCHEDULER_PERSIST = True
#
# # 配置密码
# REDIS_PARAMS = {
# 'password': REDIS_WORD,
# }
#
#
================================================
FILE: 项目/HouseScrapy/spiders/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-15 Python: 3.7
================================================
FILE: 项目/HouseScrapy/spiders/house.py
================================================
# -*- coding: utf-8 -*-
import scrapy
from settings import HOST
from toolkits.items import HousesItem
class HouseSpider(scrapy.Spider):
name = 'house'
start_urls = ['http://www.funi.com/loupan/region_0_0_0_0_1']
target_url = 'http://www.funi.com/loupan/region_0_0_0_0_{page}'
custom_settings = {
'ITEM_PIPELINES': {
'toolkits.pipelines.HousePipeline': 300,
},
'DOWNLOADER_MIDDLEWARES': {
'toolkits.proxies.ProxyMiddleware': 15, # Proxy
'toolkits.make_ua.RandomUserAgentMiddleware': 20, # random UA
},
}
def parse(self, response):
"""取总页数
"""
total_page = response.xpath('//*[@class="fleft"]/div[3]/a[9]/text()').extract_first()
try:
for page in range(1, int(total_page)):
url = self.target_url.format(page=page)
yield scrapy.Request(url, callback=self.parse_url)
except Exception:
print('获取页数失败')
def parse_url(self, response):
"""获取房产链接
"""
# 获取并清洗链接
a_list = response.xpath('//*[@class="fleft"]/div[2]/dl/dt/a')
# 访问楼盘
for a in a_list:
href = a.xpath('./@href').extract_first()
title = a.xpath('./@title').extract_first()
url = HOST + href[:href.find(';')] if ';' in href else HOST + href
# 获取基础信息
yield scrapy.Request(url + '/detail.htm', callback=self.parse_base, meta={'name': title, 'base_url': url})
def parse_base(self, response):
"""解析楼盘基础信息
"""
dls = response.xpath('//*[@class="intro"]/dl')
if len(dls) == 2:
key_base = dls[0].xpath('./dd/ul/li/span/text()').extract()
values_base = [node.strip() for node in dls[0].xpath('./dd/ul/li/em/text()').extract()]
key_info = dls[1].xpath('./dd/ul/li/span/text()').extract()
values_info = dls[1].xpath('./dd/ul/li/em/text()').extract()
# 数据压缩
data = dict(dict(zip(key_base, values_base)), **dict(zip(key_info, values_info)))
# 提取传输数据
data['楼盘名'] = response.meta.get('name')
base_url = response.meta.get('base_url')
# 获取 最新动态
yield scrapy.Request(base_url + '/news.htm', callback=self.parse_news,
meta={'base_url': base_url, 'data': data})
def parse_news(self, response):
"""解析新闻
"""
dls = response.xpath('//*[@class="tml_main"]/dl')
news = []
for dl in dls:
number = dl.xpath('./dd/div/div[1]/b/i/text()').extract_first()
ps = dl.xpath('./dd/div/div[2]/p')
strong_info = dl.xpath('./dd/div/div[2]/p[1]/em/text()').extract() # 预售
new_info = dl.xpath('./dd/div/div[2]/ul/li/text()').extract() # 最新动态
# 组装数据
info = {'商品房预售许可证': number}
# 数组清洗
for node in strong_info: # 预售信息
_str = node.split('：')
if len(_str) == 2:
info[_str[0]] = _str[1]
for node in new_info: # 预售资质
_str = node.split('：')
if len(_str) == 2:
info[_str[0]] = _str[1]
for index, p in enumerate(ps): # 户数
n = p.xpath('./text()').extract_first()
if index == 1 or not n:
continue
_str = n.split('：')
if len(_str) == 2:
info[_str[0]] = _str[1]
news.append(info)
# 组装数据
data = response.meta.get('data')
data['最新动态'] = news
base_url = response.meta.get('base_url')
yield scrapy.Request(base_url, callback=self.parse_info, meta={'base_url': base_url, 'data': data})
def parse_info(self, response):
"""小区基础数据
"""
address = response.xpath('//*[@class="pro_info"]/p[2]/i/text()').extract_first() # 楼盘地址
company = response.xpath('//*[@class="pro_info"]/p[3]/i/text()').extract_first() # 开发商
tags = response.xpath('//*[@class="pro_info"]/p[4]/span/text()').extract() # 标签
phone = response.xpath('//*[@class="pro_info"]/p[5]/b/text()').extract_first() # 售楼部电话
open_time = response.xpath('//*[@class="pro_info"]/p[6]/i/text()').extract_first() # 开盘时间
house_type = response.xpath('//*[@class="pro_info"]/p[7]/i/text()').extract_first() # 建筑类型
number = response.xpath('//*[@class="pro_info"]/p[8]/i/text()').extract_first() # 规划用户
expense = response.xpath('//*[@class="pro_info"]/p[8]/i[2]/text()').extract_first() # 物管费
expense = expense.strip() if expense else ''
phone = phone.strip() if phone else ''
data = response.meta.get('data')
data['楼盘地址'] = address
data['开发商'] = company
data['特色'] = tags
data['售楼电话'] = phone
data['开盘时间'] = open_time
data['建筑类型'] = house_type
data['规划户数'] = number
data['物管费'] = expense
data['网址'] = response.meta.get('base_url')
item = HousesItem()
item['data'] = data
yield item
================================================
FILE: 项目/HouseScrapy/toolkits/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-15 Python: 3.7
================================================
FILE: 项目/HouseScrapy/toolkits/fake_useragent.json
================================================
{
"browsers": {
"chrome": ["Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14"],
"opera": ["Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"],
"firefox": ["Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0", "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0", "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0", "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0", "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0", "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0", "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0", "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0", "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6"],
"internetexplorer": ["Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)"],
"safari": ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10", "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5"]
},
"randomize": {
"344": "chrome",
"819": "firefox",
"346": "chrome",
"347": "chrome",
"340": "chrome",
"341": "chrome",
"342": "chrome",
"343": "chrome",
"810": "internetexplorer",
"811": "internetexplorer",
"812": "internetexplorer",
"813": "firefox",
"348": "chrome",
"349": "chrome",
"816": "firefox",
"817": "firefox",
"737": "chrome",
"719": "chrome",
"718": "chrome",
"717": "chrome",
"716": "chrome",
"715": "chrome",
"714": "chrome",
"713": "chrome",
"712": "chrome",
"711": "chrome",
"710": "chrome",
"421": "chrome",
"129": "chrome",
"420": "chrome",
"423": "chrome",
"422": "chrome",
"425": "chrome",
"619": "chrome",
"424": "chrome",
"427": "chrome",
"298": "chrome",
"299": "chrome",
"296": "chrome",
"297": "chrome",
"294": "chrome",
"295": "chrome",
"292": "chrome",
"293": "chrome",
"290": "chrome",
"291": "chrome",
"591": "chrome",
"590": "chrome",
"593": "chrome",
"592": "chrome",
"595": "chrome",
"594": "chrome",
"597": "chrome",
"596": "chrome",
"195": "chrome",
"194": "chrome",
"197": "chrome",
"196": "chrome",
"191": "chrome",
"190": "chrome",
"193": "chrome",
"192": "chrome",
"270": "chrome",
"271": "chrome",
"272": "chrome",
"273": "chrome",
"274": "chrome",
"275": "chrome",
"276": "chrome",
"277": "chrome",
"278": "chrome",
"279": "chrome",
"569": "chrome",
"497": "chrome",
"524": "chrome",
"525": "chrome",
"526": "chrome",
"527": "chrome",
"520": "chrome",
"521": "chrome",
"522": "chrome",
"523": "chrome",
"528": "chrome",
"529": "chrome",
"449": "chrome",
"448": "chrome",
"345": "chrome",
"443": "chrome",
"442": "chrome",
"441": "chrome",
"440": "chrome",
"447": "chrome",
"446": "chrome",
"445": "chrome",
"444": "chrome",
"47": "chrome",
"108": "chrome",
"109": "chrome",
"102": "chrome",
"103": "chrome",
"100": "chrome",
"101": "chrome",
"106": "chrome",
"107": "chrome",
"104": "chrome",
"105": "chrome",
"902": "firefox",
"903": "firefox",
"39": "chrome",
"38": "chrome",
"906": "firefox",
"907": "firefox",
"904": "firefox",
"905": "firefox",
"33": "chrome",
"32": "chrome",
"31": "chrome",
"30": "chrome",
"37": "chrome",
"36": "chrome",
"35": "chrome",
"34": "chrome",
"641": "chrome",
"640": "chrome",
"643": "chrome",
"642": "chrome",
"645": "chrome",
"644": "chrome",
"438": "chrome",
"439": "chrome",
"436": "chrome",
"437": "chrome",
"434": "chrome",
"435": "chrome",
"432": "chrome",
"433": "chrome",
"430": "chrome",
"431": "chrome",
"826": "firefox",
"339": "chrome",
"338": "chrome",
"335": "chrome",
"334": "chrome",
"337": "chrome",
"336": "chrome",
"331": "chrome",
"330": "chrome",
"333": "chrome",
"332": "chrome",
"559": "chrome",
"745": "chrome",
"854": "firefox",
"818": "firefox",
"856": "firefox",
"857": "firefox",
"850": "firefox",
"851": "firefox",
"852": "firefox",
"0": "chrome",
"858": "firefox",
"859": "firefox",
"748": "chrome",
"6": "chrome",
"43": "chrome",
"99": "chrome",
"98": "chrome",
"91": "chrome",
"90": "chrome",
"93": "chrome",
"92": "chrome",
"95": "chrome",
"94": "chrome",
"97": "chrome",
"96": "chrome",
"814": "firefox",
"815": "firefox",
"153": "chrome",
"740": "chrome",
"741": "chrome",
"742": "chrome",
"743": "chrome",
"744": "chrome",
"558": "chrome",
"746": "chrome",
"747": "chrome",
"555": "chrome",
"554": "chrome",
"557": "chrome",
"556": "chrome",
"551": "chrome",
"550": "chrome",
"553": "chrome",
"552": "chrome",
"238": "chrome",
"239": "chrome",
"234": "chrome",
"235": "chrome",
"236": "chrome",
"237": "chrome",
"230": "chrome",
"231": "chrome",
"232": "chrome",
"233": "chrome",
"1": "chrome",
"155": "chrome",
"146": "chrome",
"147": "chrome",
"618": "chrome",
"145": "chrome",
"142": "chrome",
"143": "chrome",
"140": "chrome",
"141": "chrome",
"612": "chrome",
"613": "chrome",
"610": "chrome",
"611": "chrome",
"616": "chrome",
"617": "chrome",
"148": "chrome",
"149": "chrome",
"46": "chrome",
"154": "chrome",
"948": "safari",
"949": "safari",
"946": "safari",
"947": "safari",
"944": "safari",
"945": "safari",
"942": "safari",
"943": "safari",
"940": "safari",
"941": "safari",
"689": "chrome",
"688": "chrome",
"685": "chrome",
"684": "chrome",
"687": "chrome",
"686": "chrome",
"681": "chrome",
"680": "chrome",
"683": "chrome",
"682": "chrome",
"458": "chrome",
"459": "chrome",
"133": "chrome",
"132": "chrome",
"131": "chrome",
"130": "chrome",
"137": "chrome",
"136": "chrome",
"135": "chrome",
"134": "chrome",
"494": "chrome",
"495": "chrome",
"139": "chrome",
"138": "chrome",
"490": "chrome",
"491": "chrome",
"492": "chrome",
"493": "chrome",
"24": "chrome",
"25": "chrome",
"26": "chrome",
"27": "chrome",
"20": "chrome",
"21": "chrome",
"22": "chrome",
"23": "chrome",
"28": "chrome",
"29": "chrome",
"820": "firefox",
"407": "chrome",
"406": "chrome",
"405": "chrome",
"404": "chrome",
"403": "chrome",
"402": "chrome",
"401": "chrome",
"400": "chrome",
"933": "firefox",
"932": "firefox",
"931": "firefox",
"930": "firefox",
"937": "safari",
"452": "chrome",
"409": "chrome",
"408": "chrome",
"453": "chrome",
"414": "chrome",
"183": "chrome",
"415": "chrome",
"379": "chrome",
"378": "chrome",
"228": "chrome",
"829": "firefox",
"828": "firefox",
"371": "chrome",
"370": "chrome",
"373": "chrome",
"372": "chrome",
"375": "chrome",
"374": "chrome",
"377": "chrome",
"376": "chrome",
"708": "chrome",
"709": "chrome",
"704": "chrome",
"705": "chrome",
"706": "chrome",
"707": "chrome",
"700": "chrome",
"144": "chrome",
"702": "chrome",
"703": "chrome",
"393": "chrome",
"392": "chrome",
"88": "chrome",
"89": "chrome",
"397": "chrome",
"396": "chrome",
"395": "chrome",
"394": "chrome",
"82": "chrome",
"83": "chrome",
"80": "chrome",
"81": "chrome",
"86": "chrome",
"87": "chrome",
"84": "chrome",
"85": "chrome",
"797": "internetexplorer",
"796": "internetexplorer",
"795": "internetexplorer",
"794": "internetexplorer",
"793": "internetexplorer",
"792": "internetexplorer",
"791": "internetexplorer",
"790": "internetexplorer",
"749": "chrome",
"799": "internetexplorer",
"798": "internetexplorer",
"7": "chrome",
"170": "chrome",
"586": "chrome",
"587": "chrome",
"584": "chrome",
"585": "chrome",
"582": "chrome",
"583": "chrome",
"580": "chrome",
"581": "chrome",
"588": "chrome",
"589": "chrome",
"245": "chrome",
"244": "chrome",
"247": "chrome",
"246": "chrome",
"241": "chrome",
"614": "chrome",
"243": "chrome",
"242": "chrome",
"615": "chrome",
"249": "chrome",
"248": "chrome",
"418": "chrome",
"419": "chrome",
"519": "chrome",
"518": "chrome",
"511": "chrome",
"510": "chrome",
"513": "chrome",
"512": "chrome",
"515": "chrome",
"514": "chrome",
"517": "chrome",
"516": "chrome",
"623": "chrome",
"622": "chrome",
"621": "chrome",
"620": "chrome",
"627": "chrome",
"626": "chrome",
"625": "chrome",
"624": "chrome",
"450": "chrome",
"451": "chrome",
"629": "chrome",
"628": "chrome",
"454": "chrome",
"455": "chrome",
"456": "chrome",
"457": "chrome",
"179": "chrome",
"178": "chrome",
"177": "chrome",
"199": "chrome",
"175": "chrome",
"174": "chrome",
"173": "chrome",
"172": "chrome",
"171": "chrome",
"198": "chrome",
"977": "opera",
"182": "chrome",
"975": "opera",
"974": "opera",
"973": "opera",
"972": "opera",
"971": "opera",
"970": "opera",
"180": "chrome",
"979": "opera",
"978": "opera",
"656": "chrome",
"599": "chrome",
"654": "chrome",
"181": "chrome",
"186": "chrome",
"187": "chrome",
"184": "chrome",
"185": "chrome",
"652": "chrome",
"188": "chrome",
"189": "chrome",
"658": "chrome",
"653": "chrome",
"650": "chrome",
"651": "chrome",
"11": "chrome",
"10": "chrome",
"13": "chrome",
"12": "chrome",
"15": "chrome",
"14": "chrome",
"17": "chrome",
"16": "chrome",
"19": "chrome",
"18": "chrome",
"863": "firefox",
"862": "firefox",
"865": "firefox",
"864": "firefox",
"867": "firefox",
"866": "firefox",
"354": "chrome",
"659": "chrome",
"44": "chrome",
"883": "firefox",
"882": "firefox",
"881": "firefox",
"880": "firefox",
"887": "firefox",
"886": "firefox",
"885": "firefox",
"884": "firefox",
"889": "firefox",
"888": "firefox",
"116": "chrome",
"45": "chrome",
"657": "chrome",
"355": "chrome",
"322": "chrome",
"323": "chrome",
"320": "chrome",
"321": "chrome",
"326": "chrome",
"327": "chrome",
"324": "chrome",
"325": "chrome",
"328": "chrome",
"329": "chrome",
"562": "chrome",
"201": "chrome",
"200": "chrome",
"203": "chrome",
"202": "chrome",
"205": "chrome",
"204": "chrome",
"207": "chrome",
"206": "chrome",
"209": "chrome",
"208": "chrome",
"779": "internetexplorer",
"778": "internetexplorer",
"77": "chrome",
"76": "chrome",
"75": "chrome",
"74": "chrome",
"73": "chrome",
"72": "chrome",
"71": "chrome",
"70": "chrome",
"655": "chrome",
"567": "chrome",
"79": "chrome",
"78": "chrome",
"359": "chrome",
"358": "chrome",
"669": "chrome",
"668": "chrome",
"667": "chrome",
"666": "chrome",
"665": "chrome",
"664": "chrome",
"663": "chrome",
"662": "chrome",
"661": "chrome",
"660": "chrome",
"215": "chrome",
"692": "chrome",
"693": "chrome",
"690": "chrome",
"691": "chrome",
"696": "chrome",
"697": "chrome",
"694": "chrome",
"695": "chrome",
"698": "chrome",
"699": "chrome",
"542": "chrome",
"543": "chrome",
"540": "chrome",
"541": "chrome",
"546": "chrome",
"547": "chrome",
"544": "chrome",
"545": "chrome",
"8": "chrome",
"548": "chrome",
"549": "chrome",
"598": "chrome",
"869": "firefox",
"868": "firefox",
"120": "chrome",
"121": "chrome",
"122": "chrome",
"123": "chrome",
"124": "chrome",
"125": "chrome",
"126": "chrome",
"127": "chrome",
"128": "chrome",
"2": "chrome",
"219": "chrome",
"176": "chrome",
"214": "chrome",
"563": "chrome",
"928": "firefox",
"929": "firefox",
"416": "chrome",
"417": "chrome",
"410": "chrome",
"411": "chrome",
"412": "chrome",
"413": "chrome",
"920": "firefox",
"498": "chrome",
"922": "firefox",
"923": "firefox",
"924": "firefox",
"925": "firefox",
"926": "firefox",
"927": "firefox",
"319": "chrome",
"318": "chrome",
"313": "chrome",
"312": "chrome",
"311": "chrome",
"310": "chrome",
"317": "chrome",
"316": "chrome",
"315": "chrome",
"314": "chrome",
"921": "firefox",
"496": "chrome",
"832": "firefox",
"833": "firefox",
"830": "firefox",
"831": "firefox",
"836": "firefox",
"837": "firefox",
"834": "firefox",
"835": "firefox",
"838": "firefox",
"839": "firefox",
"3": "chrome",
"368": "chrome",
"369": "chrome",
"366": "chrome",
"367": "chrome",
"364": "chrome",
"365": "chrome",
"362": "chrome",
"363": "chrome",
"360": "chrome",
"361": "chrome",
"218": "chrome",
"380": "chrome",
"861": "firefox",
"382": "chrome",
"383": "chrome",
"384": "chrome",
"385": "chrome",
"386": "chrome",
"387": "chrome",
"388": "chrome",
"389": "chrome",
"784": "internetexplorer",
"785": "internetexplorer",
"786": "internetexplorer",
"787": "internetexplorer",
"780": "internetexplorer",
"781": "internetexplorer",
"782": "internetexplorer",
"381": "chrome",
"788": "internetexplorer",
"789": "internetexplorer",
"860": "firefox",
"151": "chrome",
"579": "chrome",
"578": "chrome",
"150": "chrome",
"573": "chrome",
"572": "chrome",
"571": "chrome",
"570": "chrome",
"577": "chrome",
"576": "chrome",
"575": "chrome",
"574": "chrome",
"60": "chrome",
"61": "chrome",
"62": "chrome",
"259": "chrome",
"64": "chrome",
"65": "chrome",
"66": "chrome",
"67": "chrome",
"68": "chrome",
"253": "chrome",
"250": "chrome",
"251": "chrome",
"256": "chrome",
"257": "chrome",
"254": "chrome",
"255": "chrome",
"499": "chrome",
"157": "chrome",
"156": "chrome",
"939": "safari",
"731": "chrome",
"730": "chrome",
"733": "chrome",
"938": "safari",
"735": "chrome",
"734": "chrome",
"508": "chrome",
"736": "chrome",
"506": "chrome",
"738": "chrome",
"504": "chrome",
"505": "chrome",
"502": "chrome",
"503": "chrome",
"500": "chrome",
"501": "chrome",
"630": "chrome",
"631": "chrome",
"632": "chrome",
"633": "chrome",
"469": "chrome",
"468": "chrome",
"636": "chrome",
"637": "chrome",
"465": "chrome",
"464": "chrome",
"467": "chrome",
"466": "chrome",
"461": "chrome",
"900": "firefox",
"463": "chrome",
"462": "chrome",
"901": "firefox",
"168": "chrome",
"169": "chrome",
"164": "chrome",
"165": "chrome",
"166": "chrome",
"167": "chrome",
"160": "chrome",
"161": "chrome",
"162": "chrome",
"163": "chrome",
"964": "safari",
"965": "safari",
"966": "safari",
"967": "safari",
"960": "safari",
"961": "safari",
"962": "safari",
"963": "safari",
"783": "internetexplorer",
"968": "safari",
"969": "opera",
"936": "firefox",
"935": "firefox",
"934": "firefox",
"908": "firefox",
"909": "firefox",
"722": "chrome",
"426": "chrome",
"878": "firefox",
"879": "firefox",
"876": "firefox",
"877": "firefox",
"874": "firefox",
"875": "firefox",
"872": "firefox",
"873": "firefox",
"870": "firefox",
"871": "firefox",
"9": "chrome",
"890": "firefox",
"891": "firefox",
"892": "firefox",
"893": "firefox",
"894": "firefox",
"647": "chrome",
"896": "firefox",
"897": "firefox",
"898": "firefox",
"899": "firefox",
"646": "chrome",
"649": "chrome",
"648": "chrome",
"357": "chrome",
"356": "chrome",
"809": "internetexplorer",
"808": "internetexplorer",
"353": "chrome",
"352": "chrome",
"351": "chrome",
"350": "chrome",
"803": "internetexplorer",
"802": "internetexplorer",
"801": "internetexplorer",
"800": "internetexplorer",
"807": "internetexplorer",
"806": "internetexplorer",
"805": "internetexplorer",
"804": "internetexplorer",
"216": "chrome",
"217": "chrome",
"768": "chrome",
"769": "chrome",
"212": "chrome",
"213": "chrome",
"210": "chrome",
"211": "chrome",
"762": "chrome",
"763": "chrome",
"760": "chrome",
"761": "chrome",
"766": "chrome",
"767": "chrome",
"764": "chrome",
"765": "chrome",
"40": "chrome",
"41": "chrome",
"289": "chrome",
"288": "chrome",
"4": "chrome",
"281": "chrome",
"280": "chrome",
"283": "chrome",
"282": "chrome",
"285": "chrome",
"284": "chrome",
"287": "chrome",
"286": "chrome",
"678": "chrome",
"679": "chrome",
"674": "chrome",
"675": "chrome",
"676": "chrome",
"677": "chrome",
"670": "chrome",
"671": "chrome",
"672": "chrome",
"673": "chrome",
"263": "chrome",
"262": "chrome",
"261": "chrome",
"260": "chrome",
"267": "chrome",
"266": "chrome",
"265": "chrome",
"264": "chrome",
"269": "chrome",
"268": "chrome",
"59": "chrome",
"58": "chrome",
"55": "chrome",
"54": "chrome",
"57": "chrome",
"56": "chrome",
"51": "chrome",
"258": "chrome",
"53": "chrome",
"52": "chrome",
"537": "chrome",
"536": "chrome",
"535": "chrome",
"63": "chrome",
"533": "chrome",
"532": "chrome",
"531": "chrome",
"530": "chrome",
"152": "chrome",
"539": "chrome",
"538": "chrome",
"775": "internetexplorer",
"774": "internetexplorer",
"982": "opera",
"983": "opera",
"980": "opera",
"981": "opera",
"777": "internetexplorer",
"984": "opera",
"50": "chrome",
"115": "chrome",
"252": "chrome",
"117": "chrome",
"776": "internetexplorer",
"111": "chrome",
"110": "chrome",
"113": "chrome",
"69": "chrome",
"771": "chrome",
"119": "chrome",
"118": "chrome",
"770": "chrome",
"773": "internetexplorer",
"772": "internetexplorer",
"429": "chrome",
"428": "chrome",
"534": "chrome",
"919": "firefox",
"918": "firefox",
"915": "firefox",
"914": "firefox",
"917": "firefox",
"916": "firefox",
"911": "firefox",
"910": "firefox",
"913": "firefox",
"912": "firefox",
"308": "chrome",
"309": "chrome",
"855": "firefox",
"300": "chrome",
"301": "chrome",
"302": "chrome",
"303": "chrome",
"304": "chrome",
"305": "chrome",
"306": "chrome",
"307": "chrome",
"895": "firefox",
"825": "firefox",
"824": "firefox",
"827": "firefox",
"847": "firefox",
"846": "firefox",
"845": "firefox",
"844": "firefox",
"843": "firefox",
"842": "firefox",
"841": "firefox",
"840": "firefox",
"821": "firefox",
"853": "firefox",
"849": "firefox",
"848": "firefox",
"823": "firefox",
"822": "firefox",
"240": "chrome",
"390": "chrome",
"732": "chrome",
"753": "chrome",
"752": "chrome",
"751": "chrome",
"750": "chrome",
"757": "chrome",
"756": "chrome",
"755": "chrome",
"754": "chrome",
"560": "chrome",
"561": "chrome",
"759": "chrome",
"758": "chrome",
"564": "chrome",
"565": "chrome",
"566": "chrome",
"701": "chrome",
"739": "chrome",
"229": "chrome",
"507": "chrome",
"227": "chrome",
"226": "chrome",
"225": "chrome",
"224": "chrome",
"223": "chrome",
"222": "chrome",
"221": "chrome",
"220": "chrome",
"114": "chrome",
"391": "chrome",
"726": "chrome",
"727": "chrome",
"724": "chrome",
"725": "chrome",
"568": "chrome",
"723": "chrome",
"720": "chrome",
"721": "chrome",
"728": "chrome",
"729": "chrome",
"605": "chrome",
"604": "chrome",
"607": "chrome",
"606": "chrome",
"601": "chrome",
"600": "chrome",
"603": "chrome",
"602": "chrome",
"159": "chrome",
"158": "chrome",
"112": "chrome",
"609": "chrome",
"608": "chrome",
"976": "opera",
"634": "chrome",
"399": "chrome",
"635": "chrome",
"959": "safari",
"958": "safari",
"398": "chrome",
"48": "chrome",
"49": "chrome",
"951": "safari",
"950": "safari",
"953": "safari",
"952": "safari",
"42": "chrome",
"954": "safari",
"957": "safari",
"956": "safari",
"638": "chrome",
"5": "chrome",
"639": "chrome",
"460": "chrome",
"489": "chrome",
"488": "chrome",
"487": "chrome",
"486": "chrome",
"485": "chrome",
"484": "chrome",
"483": "chrome",
"482": "chrome",
"481": "chrome",
"480": "chrome",
"509": "chrome",
"955": "safari",
"472": "chrome",
"473": "chrome",
"470": "chrome",
"471": "chrome",
"476": "chrome",
"477": "chrome",
"474": "chrome",
"475": "chrome",
"478": "chrome",
"479": "chrome"
}
}
================================================
FILE: 项目/HouseScrapy/toolkits/items.py
================================================
# -*- coding: utf-8 -*-
import scrapy
class HousesItem(scrapy.Item):
data = scrapy.Field()
================================================
FILE: 项目/HouseScrapy/toolkits/make_ua.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/7 Python: 3.7
import os
from fake_useragent import UserAgent
class RandomUserAgentMiddleware(object):
"""
first to use location because it is the fastest
"""
def __init__(self):
location = os.getcwd() + '/toolkits/fake_useragent.json'
self.agent = UserAgent(path=location)
# self.agent = UserAgent(verify_ssl=False)
# self.agent = UserAgent(use_cache_server=False)
@classmethod
def from_crawler(cls, crawler):
return cls()
def process_request(self, request, spider):
request.headers.setdefault('User-Agent', self.agent.random)
================================================
FILE: 项目/HouseScrapy/toolkits/middlewares.py
================================================
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class HousescrapySpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class HousescrapyDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
================================================
FILE: 项目/HouseScrapy/toolkits/pipelines.py
================================================
# -*- coding: utf-8 -*-
from pymongo import MongoClient
from bson.objectid import ObjectId
from settings import MONGODB_URL, MONGODB_DB, MONGODB_COLL
class HousePipeline(object):
"""地产基础数据
"""
def __init__(self):
client = MongoClient(MONGODB_URL)
self.coll = client[MONGODB_DB][MONGODB_COLL] # 地产链接
def process_item(self, item, spider):
self.coll.insert_one(item['data'])
================================================
FILE: 项目/HouseScrapy/toolkits/proxies.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/3/5 Python: 3.7
import base64
from settings import PROXY_USER, PROXY_PASS
# 代理服务器
proxyServer = "http://http-dyn.abuyun.com:9020"
proxyAuth = "Basic " + base64.urlsafe_b64encode(bytes((PROXY_USER + ":" + PROXY_PASS), "ascii")).decode("utf8")
class ProxyMiddleware(object):
"""自定义中间件代理IP"""
def process_request(self, request, spider):
request.meta["proxy"] = proxyServer
request.headers["Proxy-Authorization"] = proxyAuth
================================================
FILE: 项目/HouseSpider/README.md
================================================
# 目前项目还在抽空更新中
> 慢慢填坑
# 概述
> 对 `www.funi.com` 网站进行数据爬取
================================================
FILE: 项目/HouseSpider/config.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
# Redis
REDIS_HOST = '127.0.0.1'
REDIS_PORT = '6379'
REDIS_PASSWORD = None
# MongoDB
MONGO_CLEAN = 'mongodb://localhost:27017'
# TargetUrl
TARGET_URL = "http://www.funi.com/loupan/region_0_0_0_0_{page}"
# ProxyIP
PROXY_USER = ""
PROXY_PASS = ""
# HOST
HOST = 'http://www.funi.com'
================================================
FILE: 项目/HouseSpider/db/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
================================================
FILE: 项目/HouseSpider/main.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
import asyncio
from tool.parse import *
from tool.toolkit import *
async def get_max_page():
"""获取总页数
"""
url = TARGET_URL.format(page=1)
result = await get(url)
return await parse_total_page(result)
async def get_house_url(page):
"""获取地产链接
"""
url = TARGET_URL.format(page=page)
result = await get(url)
await parse_house_url(result, page)
@count_time
def main():
loop = asyncio.get_event_loop()
# 1. 获取总页数
task = loop.create_task(get_max_page())
total_page = loop.run_until_complete(task)
# 2. 获取链接
house_url_func = [asyncio.ensure_future(get_house_url(_)) for _ in range(1, int(total_page))]
loop.run_until_complete(asyncio.wait(house_url_func))
# 3. 楼盘详情
if __name__ == '__main__':
main()
================================================
FILE: 项目/HouseSpider/tool/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
================================================
FILE: 项目/HouseSpider/tool/parse.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
from pyquery import PyQuery as pq
from config import *
async def parse_total_page(result):
"""解析总页数
"""
doc = pq(result)
max_page = doc('.pages a').eq(-2).text()
print('数据总: {total} 页'.format(total=max_page))
return max_page
async def parse_house_url(result, page):
"""页面解析链接
"""
doc = pq(result)
dls = doc('.fleft div').eq(-2)('dl')
n = 0
for dl in dls:
href = pq(dl)('dt a').attr('href')
href = HOST + href[: href.find(';')] # 清洗链接
print(href)
n += 1
if not n:
print('第 {page} 抽取链接失败'.format(page=page))
================================================
FILE: 项目/HouseSpider/tool/proxy.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-12 Python: 3.7
from config import PROXY_PASS, PROXY_USER
# 代理服务器
proxyHost = "http-dyn.abuyun.com"
proxyPort = "9020"
proxyServer = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
"host": proxyHost,
"port": proxyPort,
"user": PROXY_USER,
"pass": PROXY_PASS,
}
if not PROXY_USER or not PROXY_PASS:
msg = """
请先在 config.py 配置文件内填入代理IP账号
阿布云代理IP：https://www.abuyun.com/http-proxy/products.html
"""
print(msg)
exit()
================================================
FILE: 项目/HouseSpider/tool/toolkit.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-07-13 Python: 3.7
import datetime
import aiohttp
from tool.proxy import proxyServer
async def get(url):
"""请求页面
"""
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
'Host': 'www.funi.com'
}
try:
"""conn = aiohttp.TCPConnector(verify_ssl=False) connector=conn"""
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url, proxy=proxyServer) as response:
return await response.text("utf-8")
except TimeoutError as te:
print('超时', te)
def count_time(func):
"""取运行时间
"""
def int_time(*args, **kwargs):
start_time = datetime.datetime.now() # 程序开始时间
func()
over_time = datetime.datetime.now() # 程序结束时间
total_time = (over_time-start_time).total_seconds()
print('程序耗时: %s 秒' % total_time)
return int_time
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/__init__.py
================================================
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/items.py
================================================
# -*- coding: utf-8 -*-
import scrapy
class AreaItem(scrapy.Item):
"""地区"""
type = scrapy.Field()
id = scrapy.Field()
pid = scrapy.Field()
name = scrapy.Field()
pinyin = scrapy.Field()
first = scrapy.Field()
haschild = scrapy.Field()
class CoordItem(scrapy.Item):
"""坐标录入"""
type = scrapy.Field()
id = scrapy.Field()
lng = scrapy.Field()
lat = scrapy.Field()
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/middlewares.py
================================================
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class MeituanareaSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class MeituanareaDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/pipelines.py
================================================
# -*- coding: utf-8 -*-
import pymysql
from MeiTuanArea.settings import MYSQL_DB_NAME, MYSQL_HOST, MYSQL_PASSWORD, MYSQL_PORT, MYSQL_USER
class AreaPipeline(object):
"""区域采集"""
conn = None
cursor = None # 游标对象
def open_spider(self, spider):
print('开始爬虫，链接数据库')
self.conn = pymysql.Connect(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB_NAME,
)
def process_item(self, item, spider):
sql = None
if item['type'] == 'province': # 一级 省市
sql = """INSERT INTO province(id, `name`, pinyin, `first`) VALUES ("{id}","{name}","{pinyin}","{first}") """.format(
id=int(item['id']),
name=item['name'],
pinyin=item['pinyin'],
first=item['first'],
)
elif item['type'] == 'city': # 二级 市
sql = """INSERT INTO city(id,pid, `name`, pinyin, `first`) VALUES ("{id}","{pid}","{name}","{pinyin}","{first}") """.format(
id=int(item['id']),
pid=int(item['pid']),
name=item['name'],
pinyin=item['pinyin'],
first=item['first'],
)
elif item['type'] == 'area': # 三级 区域
sql = """INSERT INTO area(id,pid, `name`, pinyin, `first`, haschild) VALUES ("{id}","{pid}","{name}","{pinyin}","{first}","{haschild}") """.format(
id=int(item['id']),
pid=int(item['pid']),
name=item['name'],
pinyin=item['pinyin'],
first=item['first'],
haschild=item['haschild'],
)
elif item['type'] == 'address': # 四季 地区
sql = """INSERT INTO address(id,pid, `name`, pinyin, `first`) VALUES ("{id}","{pid}","{name}","{pinyin}","{first}") """.format(
id=int(item['id']),
pid=int(item['pid']),
name=item['name'],
pinyin=item['pinyin'],
first=item['first'],
)
# 创建游标对象
self.cursor = self.conn.cursor()
# 提交事务
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
print(e)
print('异常回滚')
self.conn.rollback()
self.cursor.close()
return item
def close_spider(self, spider):
print('爬虫结束, 关闭通道')
self.conn.close()
class CoordPipeline(object):
"""坐标采集更新"""
conn = None
cursor = None # 游标对象
def open_spider(self, spider):
print('开始爬虫，链接数据库')
self.conn = pymysql.Connect(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB_NAME,
)
def process_item(self, item, spider):
sql = None
if item['type'] == 'province': # 一级 省市
sql = """UPDATE province SET lng={lng}, lat={lat} where id={id}""".format(
id=int(item['id']),
lng=item['lng'],
lat=item['lat'],
)
elif item['type'] == 'city': # 二级 市
sql = """UPDATE city SET lng={lng}, lat={lat} where id={id}""".format(
id=int(item['id']),
lng=item['lng'],
lat=item['lat'],
)
elif item['type'] == 'area': # 三级 区域
sql = """UPDATE area SET lng={lng}, lat={lat} where id={id}""".format(
id=int(item['id']),
lng=item['lng'],
lat=item['lat'],
)
elif item['type'] == 'address': # 四季 地区
sql = """UPDATE address SET lng={lng}, lat={lat} where id={id}""".format(
id=int(item['id']),
lng=item['lng'],
lat=item['lat'],
)
# 创建游标对象
self.cursor = self.conn.cursor()
# 提交事务
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
print(e)
print('异常回滚')
self.conn.rollback()
self.cursor.close()
return item
def close_spider(self, spider):
print('爬虫结束, 关闭通道')
self.conn.close()
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/settings.py
================================================
# -*- coding: utf-8 -*-
BOT_NAME = 'MeiTuanArea' # 爬虫项目名
SPIDER_MODULES = ['MeiTuanArea.spiders'] # 爬虫目录设定
NEWSPIDER_MODULE = 'MeiTuanArea.spiders' # 爬虫生成目录
ROBOTSTXT_OBEY = False # 否认协议
RANDOMIZE_DOWNLOAD_DELAY = True # 开启随机增加毫秒级延迟，增加访问成功率
DOWNLOAD_FAIL_ON_DATALOSS = False # 重试处理
DOWNLOAD_TIMEOUT = 5 # 设置超时时间，避免ip失效等待时间过长
# HTTPERROR_ALLOWED_CODES = [301] # 禁止301
# 指定终端输出日志、日志位置
# LOG_LEVEL = 'WARNING'
# LOG_FILE = 'error_log.txt'
HTTPERROR_ALLOWED_CODES = [403]
# UA
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
# mysql
MYSQL_HOST = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'mysql 密码'
MYSQL_DB_NAME = 'mysql库'
# API 百度地图坐标获取API，申请后填写即可
API_AK = '百度地图 api ak'
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/spiders/__init__.py
================================================
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/spiders/area_coord.py
================================================
# -*- coding: utf-8 -*-
import scrapy
import pymysql
import json
from MeiTuanArea.settings import API_AK
from MeiTuanArea.settings import MYSQL_DB_NAME, MYSQL_HOST, MYSQL_PASSWORD, MYSQL_PORT, MYSQL_USER
from MeiTuanArea.items import CoordItem
class GetLngSpider(scrapy.Spider):
name = 'area_coord'
# 独立配置
custom_settings = {
'ITEM_PIPELINES': {
'MeiTuanArea.pipelines.CoordPipeline': 300,
},
}
# mysql 配置
conn = pymysql.Connect(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB_NAME,
)
url = 'http://api.map.baidu.com/geocoder/v2/?address={address}&output=json&ak={ak}'
def start_requests(self):
# 一级区域 省市
provinces = self.get_db("""SELECT id,`name` from province""")
for _id, name in provinces:
target_url = self.url.format(address=name, ak=API_AK)
yield scrapy.Request(target_url, meta={'type': 'province', '_id': _id})
# 二级区域 城市
city = self.get_db("""SELECT id,`name` from city""")
for _id, name in city:
target_url = self.url.format(address=name, ak=API_AK)
yield scrapy.Request(target_url, meta={'type': 'city', '_id': _id})
# 三级区域 区域
area = self.get_db("""select area.id, city.name, area.name from city LEFT JOIN area on city.id=area.pid""")
for _id, name, address_name in area:
address = str(name)+str(address_name)
target_url = self.url.format(address=address, ak=API_AK)
yield scrapy.Request(target_url, meta={'type': 'area', '_id': i[0]})
# 四级区域 街道
address = self.get_db("""select address.id,area.name, address.name from area LEFT JOIN address on address.pid=area.id""")
for _id, name, address_name in address:
target_url = self.url.format(address=str(name)+str(address_name), ak=API_AK)
yield scrapy.Request(target_url, meta={'type': 'address', '_id': _id})
def get_db(self, sql):
"""数据库查询"""
# 创建游标对象
cursor = self.conn.cursor()
# 提交事务
try:
cursor.execute(sql)
data = cursor.fetchall()
cursor.close()
self.conn.close()
return data
except Exception as e:
print('异常回滚')
self.conn.rollback()
cursor.close()
self.conn.close()
return None
def parse(self, response):
"""清洗数据"""
item = CoordItem()
data = json.loads(response.text)
# 处理字符串 把闲杂符号去掉
if data.get('status') == 0:
# 坐标
item['lng'] = data.get('result').get('location').get('lng')
item['lat'] = data.get('result').get('location').get('lat')
item['id'] = response.meta.get('_id')
item['type'] = response.meta.get('type')
yield item
================================================
FILE: 项目/MeiTuanArea/MeiTuanArea/spiders/areas.py
================================================
# -*- coding: utf-8 -*-
import scrapy
import json
import re
from pypinyin import pinyin, lazy_pinyin
from MeiTuanArea.items import AreaItem
class GetAreaSpider(scrapy.Spider):
name = 'areas'
# 独立配置
custom_settings = {
'ITEM_PIPELINES': {
'MeiTuanArea.pipelines.AreaPipeline': 300,
},
'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
'DOWNLOAD_DELAY': 0.5, # 限流 下载同一个网站下一个页面前需要等待的时间
}
def start_requests(self):
start_url = 'https://www.meituan.com/ptapi/getprovincecityinfo/'
yield scrapy.Request(start_url, callback=self.parse_province)
def parse_province(self, response):
"""省市+市 1、2 级区域采集"""
target_url = 'http://{acronym}.meituan.com/meishi/'
item = AreaItem()
data = json.loads(response.text)
for node in data:
name = node.get('provinceName')
item['type'] = 'province'
item['haschild'] = 1
item['id'] = node.get('provinceCode')
item['pid'] = 0
item['name'] = name
item['pinyin'] = ''.join(lazy_pinyin(name))
item['first'] = self.get_acronym(name)
yield item # 一级省市
for i in node.get('cityInfoList'):
item['type'] = 'city'
item['id'] = i.get('id')
item['pid'] = node.get('provinceCode')
item['name'] = i.get('name')
item['pinyin'] = i.get('pinyin')
item['first'] = i.get('acronym')
yield item # 二级市
url = target_url.format(acronym=i.get('acronym'))
yield scrapy.Request(url, callback=self.parse_area, meta={'pid': i.get('id')})
def parse_area(self, response):
"""区域+街道 2、3 级区域采集"""
info, areas = re.search(r',"areas":(.*?),"dinnerCountsAttr', response.text), None
if info:
areas = json.loads(info.group(1))
if areas:
city_id = response.meta.get('pid')
item = AreaItem()
# 解析区域 3 级
for area in areas:
item['type'] = 'area'
item['id'] = area.get('id')
item['pid'] = city_id
item['name'] = area.get('name')
item['pinyin'] = ''.join(lazy_pinyin(area.get('name')))
item['first'] = self.get_acronym(area.get('name'))
subs = area.get('subAreas')
# 判断是否有下级，有的区域么有下级了
if len(subs) > 1:
item['haschild'] = 1
else:
item['haschild'] = 0
yield item
# 解析 4 级
if len(subs) > 1:
for sub in subs:
if not sub.get('name') == '全部':
item['haschild'] = 0
item['type'] = 'address'
item['id'] = sub.get('id')
item['pid'] = area.get('id')
item['name'] = sub.get('name')
item['pinyin'] = ''.join(lazy_pinyin(sub.get('name')))
item['first'] = self.get_acronym(sub.get('name'))
yield item
else:
print('区域读取失败')
@staticmethod
def get_acronym(str_data):
"""
获取字符串的首字母
:param str_data: 字符串
:return: 字符串
"""
return "".join([i[0][0] for i in pinyin(str_data)])
================================================
FILE: 项目/MeiTuanArea/README.md
================================================
# 美团城市采集
> 因为全站爬取需要用到，区域基础数据。这里单独抽离出来。
## 配置
在 settings 内配置 mysql 与 百度api_ak 即可
## 数据库设计
> 因为最终数据将会用到Mysql上，区域一共有4个层级，分别是省市、市、区域、街道，这里按照业务需求拆分到4张表中。
## 坐标拾取
> 通过百度API调用地址，获取坐标并存入库中
## 效果
================================================
FILE: 项目/MeiTuanArea/__init__.py
================================================
# -*- coding: utf-8 -*-
# __author__ = "zok" 362416272@qq.com
# Date: 2019-06-18 Python: 3.7
================================================
FILE: 项目/MeiTuanArea/scrapy.cfg
================================================
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = MeiTuanArea.settings
[deploy]
#url = http://localhost:6800/
project = MeiTuanArea
================================================
FILE: 项目/MeiTuanArea/初始化.sql
================================================
/*
Navicat Premium Data Transfer
Source Server : LocalhostMysql
Source Server Type : MySQL
Source Server Version : 50725
Source Host : localhost:3306
Source Schema : nujiang
Target Server Type : MySQL
Target Server Version : 50725
File Encoding : 65001
Date: 23/05/2019 16:32:56
*/
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for address
-- ----------------------------
DROP TABLE IF EXISTS `address`;
CREATE TABLE `address` (
`id` bigint(10) NOT NULL AUTO_INCREMENT COMMENT 'ID',
`pid` bigint(10) DEFAULT NULL COMMENT '父id',
`name` varchar(100) DEFAULT NULL COMMENT '名称',
`pinyin` varchar(100) DEFAULT NULL COMMENT '拼音',
`code` varchar(100) DEFAULT NULL COMMENT '长途区号',
`zip` varchar(100) DEFAULT NULL COMMENT '邮编',
`first` varchar(50) DEFAULT NULL COMMENT '首字母',
`lng` varchar(100) DEFAULT NULL COMMENT '经度',
`lat` varchar(100) DEFAULT NULL COMMENT '纬度',
PRIMARY KEY (`id`) USING BTREE,
KEY `pid` (`pid`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=3749 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='四级区域 地址';
-- ----------------------------
-- Table structure for area
-- ----------------------------
DROP TABLE IF EXISTS `area`;
CREATE TABLE `area` (
`id` bigint(10) NOT NULL AUTO_INCREMENT COMMENT 'ID',
`pid` bigint(10) DEFAULT NULL COMMENT '父id',
`name` varchar(100) DEFAULT NULL COMMENT '名称',
`pinyin` varchar(100) DEFAULT NULL COMMENT '拼音',
`code` varchar(100) DEFAULT NULL COMMENT '长途区号',
`zip` varchar(100) DEFAULT NULL COMMENT '邮编',
`first` varchar(50) DEFAULT NULL COMMENT '首字母',
`lng` varchar(100) DEFAULT NULL COMMENT '经度',
`lat` varchar(100) DEFAULT NULL COMMENT '纬度',
`haschild` int(1) DEFAULT NULL COMMENT '是否有下级',
PRIMARY KEY (`id`) USING BTREE,
KEY `pid` (`pid`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=39793 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='三级区域 区域';
-- ----------------------------
-- Table structure for city
-- ----------------------------
DROP TABLE IF EXISTS `city`;
CREATE TABLE `city` (
`id` bigint(10) NOT NULL AUTO_INCREMENT COMMENT 'ID',
`pid` bigint(10) DEFAULT NULL COMMENT '父id',
`name` varchar(100) DEFAULT NULL COMMENT '名称',
`pinyin` varchar(100) DEFAULT NULL COMMENT '拼音',
`code` varchar(100) DEFAULT NULL COMMENT '长途区号',
`zip` varchar(100) DEFAULT NULL COMMENT '邮编',
`first` varchar(50) DEFAULT NULL COMMENT '首字母',
`lng` varchar(100) DEFAULT NULL COMMENT '经度',
`lat` varchar(100) DEFAULT NULL COMMENT '纬度',
PRIMARY KEY (`id`) USING BTREE,
KEY `pid` (`pid`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=8002 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='二级区域 城市';
-- ----------------------------
-- Table structure for province
-- ----------------------------
DROP TABLE IF EXISTS `province`;
CREATE TABLE `province` (
`id` bigint(10) NOT NULL AUTO_INCREMENT COMMENT 'ID',
`name` varchar(100) DEFAULT NULL COMMENT '名称',
`pinyin` varchar(100) DEFAULT NULL COMMENT '拼音',
`code` varchar(100) DEFAULT NULL COMMENT '长途区号',
`zip` varchar(100) DEFAULT NULL COMMENT '邮编',
`first` varchar(50) DEFAULT NULL COMMENT '首字母',
`lng` varchar(100) DEFAULT NULL COMMENT '经度',
`lat` varchar(100) DEFAULT NULL COMMENT '纬度',
PRIMARY KEY (`id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=820001 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='一级区域 省市';
SET FOREIGN_KEY_CHECKS = 1;
================================================
FILE: 项目/README.md
================================================
# 该板块不定期更新
> 因为工作中会经常开发重型的爬虫，并且也属于公司的资源，所以并不会将代码放到网上。尽量以一些实战demo形式发布一些个人小项目。
## MeiTuanArea
美团区域 Scrapy 爬虫
## HoseSpider
房地产爬虫 aiohttp 爬虫
