Pyppeteer常用技巧

Pyppeteer常用技巧

绕过检测步骤

# 在from pyppeteer import launch 导入前，执行以下代码 from pyppeteer import launcher launcher.DEFAULT_ARGS.remove("--enable-automation") from pyppeteer_stealth import stealth   # 先安装pyppeteer_stealth ##################################################################### # 或点开源码launcher将‘--enable-automation’ 注释掉 from pyppeteer_stealth import stealth    

# 先进入搜索页面 async def search_page(url,cookie_):     browser = await launch({ 
        'headless': False,'dumpio':True, 'autoClose':False})     page = await browser.newPage()     await stealth(page)  # <-- Here     await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3542.0 Safari/537.36')     await page.evaluateOnNewDocument('() =>{ Object.defineProperties(navigator,'                       '{ webdriver:{ get: () => false } }) }') # await page.setViewport({"width": 1920, "height": 1080}) # 设置窗口大小     await page.setViewport({ 
        "width": 1100, "height": 768})     cookie_list = await get_cookie(cookie_)     for cookie in cookie_list:         await page.setCookie(cookie) # print(cookie_list)     await page.goto(url)
    # 等待页面加载
    await page.waitForXPath('//div[@class="check_box "]')
    return page

常用方法

page_text = await page.content()

# 等待页面加载,使用xpath方式等待某元素出现
await page.waitForXPath('//div[@class="check_box "]')
# 等待某选择器出现，返回选择器，‘#’ 代表标签id属性,'.'代表标签class属性
await page.waitForSelector('#authImg')

• 输入框输入内容操作

# 使用执行js（evaluate方法）方式清空输入框
await page.evaluate('document.querySelector("#select-key:shenqingh").value=""')
# 输入内容（type方法），第一个参数是输入框选择器属性 ,输入框 #代表id '.'代表class 第二个参数是输入值
await page.type('#select-key:shenqingh', num)
 # 点击查询
await page.click("#query")

• 截图

# egg 截图某页面上的验证码，根据选择器
yazhengma = await page.waitForSelector('#authImg')
# 根据选择器截图(screenshot方法)，存储以字典形式
await yazhengma.screenshot({ 
        'path': 'yazhengma.png'})

• 设置cookie, 和selenium设置cookie类似

# cookie 格式
''' [{'domain': '.cpquery.cnipa.gov.cn', 'expiry': 2153740843, 'httpOnly': False, 'name': 'UR3ZMlLdcLIE80T', 'path': '/', 'secure': False, 'value': '4ReYPoTfhE8XU8tWBTzIJNqujckzb1ZfkSs3HZNOlhxzOsDfyC8JRlKexOUy.mK9u5bSgRIS8hRnZPIW3C8c_a9YliJfuQX7ZKI7gwK6KGNgnLyKuks2jcwfqYdg5ERFwADXj0gJ6hPvGlO6V2UV_wk_gzqcSOpsBN2jh2kBz5bEIFxfktH5PzKJRnpDBml5u4OoQMRUhzvyVVnDLiSHnqt1I60OhoMrEAuFcjl.eJbal3.cJrnvm9wlRgSz0YRrUw60O2udav7F8PZz3g8qcGBOO76lYR9mVFtJlSNNdewpSDP2G_kEwUlxBVwb65sVHPxRN0C1H.r6qURabLfzzbTpVzZXb8rAvptSmovQttf5YoA'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': '_gscs_930750436', 'path': '/', 'secure': False, 'value': '38407965zb8ett35|pv:2'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': 'bg6', 'path': '/', 'secure': False, 'value': '384|A8R2v'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': 'UR3ZMlLdcLIE83T', 'path': '/', 'secure': False, 'value': '4bg6QmpZtyIgQPzflQesXqaw5Vpns_EpqgO5UM2HwmRPegQCdj847WIMwSm9f.HL.bPxEp0Oab6kHo5v56yQPNb5rEbDoQBuqHWZ8mL8rUU3NIbT6EValIiLcD2DjTSxGBeHwVHM1P9UsYW4fXbtFYtSNRGGx3HLjPu7uT5nVuCyNVTMrQs_QTJhByLyj1bn0LRlrJItt1JgrQ.eEwBAm4Kp.z8akO_uwhobTbLenvVAVxk.kosi_iStsK_5ZjcdIUNKp3kKLUun8TZ_M3PtUTDE..qhL06PgNBZROjfDbDA.kWsLnR1SFHC_fKqeg9DSvvEqU0FBNLcwaSnYpIDe0_jMdpHCakWAE.VpZjQzi39ytpKLVbTO_z0xEaUy133ZzEV'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': 'JSESSIONID', 'path': '/', 'secure': False, 'value': '8b98d1eba495b3b5712735dcb4d5'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': 'UR3ZMlLdcLIE83S', 'path': '/', 'secure': False, 'value': 'tiIZjLZOZ9efKrNyHrF9bWq8SkX6mzf3cj7QUhCAMw6nFmofbL_wRL.5nkPFGTUU'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': '_gscu_930750436', 'path': '/', 'secure': False, 'value': '33768864tauoxr35'}, {'domain': 'cpquery.cnipa.gov.cn', 'expiry': 1953769688, 'httpOnly': False, 'name': 'UR3ZMlLdcLIE80T', 'path': '/', 'secure': False, 'value': '4JgVnsaGMQMBotHJdfL8wemn_KD7CDyGs2KRIL7PVIW7uokGNPMWxAsZR61g4ps2OjOaLJFaiIQu7CFJEPMkj9fVVKscrkOvhigVsgF4bATZQ07NUMmcaXKvhdCm89Kx5vhSyNVDIHSm.Z.is2tS9HwUG6yPrbKR9UrKZ1tJuWMcINxfK7RfN1fcftR7_POcXmQgF4pMY8GpKCd_M3oRBbw3VDYSxPO.NGGK0VOYHoEPhqJqAEENUEztie.ZdOtDaMpPDiEs1Qrh2fhUk_B9N..bf2md5LsfApiaTdOi9LwJ7Esg.i23xuIiW7cw2cLvc_wG'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': '_gscbrs_930750436', 'path': '/', 'secure': False, 'value': '1'}, {'domain': '.cpquery.cnipa.gov.cn', 'httpOnly': False, 'name': 'UR3ZMlLdcLIE80S', 'path': '/', 'secure': False, 'value': 'bmK.oV43x2p_uorGA2a8RkbLAERicnQcqpl8YMZ1OiH3RL.Gbh4QAOyqLMMSE3Mk'}] '''
# pyppeteer 设置cookie，需要遍历上面cookie列表插入
for cookie in cookie_list:
    await page.setCookie(cookie)

# selenium设置 ,也需要一个一个添加
for i in cookie_list:
    bro.add_cookie(i)