diff --git a/agency/cdn_utils.py b/agency/cdn_utils.py index a00dc17..77698eb 100755 --- a/agency/cdn_utils.py +++ b/agency/cdn_utils.py @@ -71,56 +71,8 @@ class CDNProxy: # print(i.replace("\n", "")) if i and "kyfw.12306.cn:443" not in i: cdn.append(i.replace("\n", "")) - print(cdn) return cdn - def get_cdn_list(self): - """ - 筛选代理 - :return: - """ - f = self.open_cdn_file() - url = self.urlConf["cdn_list"]["req_url"] - num = 1 - f.seek(0) - f.truncate() - for guid in self.city_list: - data = {"guid": guid, - "host": "kyfw.12306.cn", - "ishost": 0, - "encode": "HJXhdRqjh5yCF6G/AZ6EDk9faB1oSk5r", - "checktype": 0} - try: - cdn_info = self.httpClint.post(url, data, headers=self._set_header(), timeout=self.timeout).content - print(cdn_info) - if cdn_info: - split_cdn = cdn_info.split("(")[1].rstrip(")").replace("{", "").replace("}", "").split(",") - local_dict = collections.OrderedDict() - for i in split_cdn: - splits = i.split(":") - local_dict[splits[0]] = splits[2] if splits[0] == "result" else splits[1] - if local_dict and "state" in local_dict and local_dict["state"] == "1": - if "responsetime" in local_dict and local_dict["responsetime"].find("毫秒") != -1 and int(filter(str.isdigit, local_dict["responsetime"])) < 100: - f.write(json.dumps(local_dict)+"\n") - num += 1 - except Exception as e: - print(e) - print(u"本次cdn获取完成,总个数{0}".format(num)) - - def all_cdn(self): - """获取cdn列表""" - with open('cdn_list', 'r') as f: - cdn = f.readlines() - return cdn - - def par_csv(self): - cdn_csv = csv.reader(open("../cdn1.csv", "r")) - for c in cdn_csv: - cdn_re = re.compile(r'https://(\S+)/otn/index/init') - cdn_ip = re.findall(cdn_re, c[0]) - if cdn_ip and c[2] == "200": - print(cdn_ip[0]) - if __name__ == '__main__': cdn = CDNProxy() diff --git a/init/select_ticket_info.py b/init/select_ticket_info.py index 455eb38..a709e2d 100755 --- a/init/select_ticket_info.py +++ b/init/select_ticket_info.py @@ -153,7 +153,7 @@ class select: rep = http.send(urls) if rep and "message" not in rep and (datetime.datetime.now() - start_time).microseconds / 1000 < 500: if cdn[i].replace("\n", "") not in self.cdn_list: # 如果有重复的cdn,则放弃加入 - print(u"加入cdn {0}".format(cdn[i].replace("\n", ""))) + # print(u"加入cdn {0}".format(cdn[i].replace("\n", ""))) self.cdn_list.append(cdn[i].replace("\n", "")) print(u"所有cdn解析完成...") diff --git a/myUrllib/httpUtils.py b/myUrllib/httpUtils.py index cdf854b..879dcd8 100755 --- a/myUrllib/httpUtils.py +++ b/myUrllib/httpUtils.py @@ -6,6 +6,7 @@ from time import sleep import requests from config import logger + def _set_header_default(): header_dict = OrderedDict() header_dict["Accept"] = "application/json, text/plain, */*" @@ -25,6 +26,7 @@ class HTTPClient(object): """ self.initS() self._cdn = None + self._proxies = None def initS(self): self._s = requests.Session() @@ -126,7 +128,10 @@ class HTTPClient(object): try: # sleep(urls["s_time"]) if "s_time" in urls else sleep(0.001) sleep(s_time) - requests.packages.urllib3.disable_warnings() + try: + requests.packages.urllib3.disable_warnings() + except: + pass response = self._s.request(method=method, timeout=2, url="https://" + url_host + req_url,