from Doraemon import requests_dora
url = "https://www.baidu.com"

headers = requests_dora.get_default_headers()
headers["User-Agent"] = requests_dora.get_random_user_agent()

def get_proxies():
    proxy_str = "127.0.0.1:1080"
    proxies = {"http": "http://%s" % proxy_str,
               "https": "http://%s" % proxy_str, }
    return proxies

# max_times, get_proxies_fun, and invoked_by are optional parameters, others are the same as the requests.get() and requests.post()
res1 = requests_dora.try_best_2_get(url, max_times=5, get_proxies_fun=get_proxies, invoked_by="parent_fun_name") 
res2 = requests_dora.try_best_2_post(url, max_times=5, get_proxies_fun=get_proxies)
print(res1.status_code)
print(res2.status_code)

from Doraemon import proxies_dora
proxies1 = proxies_dora.get_proxies("127.0.0.1:223") # get a self-defined proxies dict
proxies2 = proxies_dora.get_data5u_proxies("your data5u api key") # input api key for crawling, get a proxies dict

pool = [
"127.0.0.1:233",
"123.123.123.123:123",
"...",
]

proxies_dora.set_pool(pool) # set a self-defined proxy pool
proxies3 = proxies_dora.get_proxies_fr_pool() # get a proxies dict from the pool

loc_info1 = proxies_dora.loc_proxy_ipv4(proxies1) # get location info of a given proxy, ipv4 only
loc_info2 = proxies_dora.loc_proxy(proxies2) # get location info of a given proxy, for both ipv4 and ipv6

from Doraemon import chrome_dora

proxy = "127.0.0.1:1080"
baidu_url = "https://www.baidu.com"
# no_images: do not load images(response more quickly)
# headless: make the chrome invisible
# proxy: set if you need
# they are all optional
chrome = chrome_dora.MyChrome(headless=False, proxy="127.0.0.1:1080", no_images=True) 
chrome.get(baidu_url)
print(chrome.page_source)

from Doraemon import google_KG

def get_proxies():
    proxy_str = "127.0.0.1:1080"
    proxies = {"http": "http://%s" % proxy_str,
               "https": "http://%s" % proxy_str, }
    return proxies

res = google_KG.get_entity("alibaba", get_proxies_fun=get_proxies)
print(res)

from Doraemon import google_translator, proxies_dora

def get_proxies():
    proxy_str = "127.0.0.1:1080"
    proxies = {"http": "http://%s" % proxy_str,
               "https": "http://%s" % proxy_str, }
    return proxies

ori_text = "中华民国"
# sl, tl and get_proxies_fun are optional, the default values are "auto", "en", None
res1 = google_translator.trans(ori_text,sl="auto", tl="zh-TW", get_proxies_fun=get_proxies) 
# replace the function get_proxies with proxies_dora.get_proxies("127.0.0.1:1080")
res2 = google_translator.trans(ori_text,sl="auto", tl="zh-TW", get_proxies_fun=lambda: proxies_dora.get_proxies("127.0.0.1:1080")) 

long_text = ori_text * 2500 # 10000 characters
res3 = google_translator.trans_long(long_text)# if len(text) > 5000

print(res1)
print(res2)

from Doraemon import dianping, proxies_dora
import json

# get_proxies_fun is optional, set if you want to use a proxy
shop_list = dianping.search_shops("2", "4s店", 1, get_proxies_fun=lambda: proxies_dora.get_proxies("127.0.0.1:1080")) # args: city id, keyword, page index
print(json.dumps(shop_list, indent=2, ensure_ascii=False))
# [{"name": "shopname1", "shop_id": "1245587}, ...]

# get_proxies_fun is optional, set if you want to use a proxy, this example use data5u proxy, 
# the website is :http://www.data5u.com/api/doc-dynamic.html 
shop_list_around = dianping.get_around("2", "5724615", 2000, 1, get_proxies_fun=lambda: proxies_dora.get_data5u_proxies("your data5u api key")) # args: city id, shop id, max distance, page index
print(json.dumps(shop_list_around, indent=2, ensure_ascii=False))
'''
shop_list_around is like this:
    [
      {
        "img_src": "https://img.meituan.net/msmerchant/2e5787325ba4579ec2e2e3f45038ade1149446.jpg%40340w_255h_1e_1c_1l%7Cwatermark%3D1%26%26r%3D1%26p%3D9%26x%3D2%26y%3D2%26relative%3D1%26o%3D20",
        "title": "速度披萨(华贸城店)",
        "star_level": 4.5,
        "review_num": 30,
        "mean_price": 89,
        "cat": "西餐",
        "region": "北苑家园",
        "addr": "清苑路13号",
        "rec_dish": [
          "黑芝麻沙拉",
          "蟹肉意面",
          "火腿榴莲披萨双拼"
        ],
        "score": {
          "taste": 8.5,
          "env": 8.4,
          "service": 8.4
        }
      },
    ]
'''

import os
from Doraemon import qq_music_crawler_by_album as qm_album, qq_music_crawler_by_area as qm_area
# crawl lyrics of songs in specific areas
area_list = ["港台", "内地"] # {'全部': -100, '内地': 200, '港台': 2, '欧美': 5, '日本': 4, '韩国': 3, '其他': 6}
save_path = "./qq_music_songs_by_area"
if not os.path.exists(save_path):
    os.makedirs(save_path)
qm_area.crawl_songs(area_list, save_path)

# crawl lyrics by albums
import json
from tqdm import tqdm

save_path = "./qq_music_songs_by_album"
if not os.path.exists(save_path):
    os.makedirs(save_path)

for sin in range(0, 7050, 30):
    ein = sin + 29
    album_list = qm_album.get_album_list(sin, ein) # get 30 albums

    for album in album_list:
        dissname = album["dissname"]
        song_list = qm_album.get_song_list(album["dissid"])
        chunk = []
        for song in tqdm(song_list, desc = "getting songs in {}".format(dissname)):
            contributors, lyric = qm_album.get_lyric(song)
            song["lyric"] = lyric
            chunk.append(song)

        json.dump(chunk, open("{}/lyric_{}.json".format(save_path, dissname), "w", encoding = "utf-8"), ensure_ascii = False)

from Doraemon import whois
ip_list = ["154.17.24.36", "154.17.24.37", "154.17.24.39", "154.17.21.36"] * 100
# # friendly
# res = whois.extract_org_names_friendly(ip_list, block_size = 100, sleep = 2)

# no limited
res = whois.extract_org_names_no_limited(ip_list)
print(res)

from Doraemon import domain2ip
threads = 100
max_fail_num = 0
domain_name2ip = {} # results
url_list = ["https://www.baidu.com", "https://www.qq.com"]
domain2ip.gethostbyname_fast(url_list, domain_name2ip, threads, max_fail_num)