{"id":1020,"date":"2022-06-19T22:12:40","date_gmt":"2022-06-19T14:12:40","guid":{"rendered":"http:\/\/175.24.184.226\/?p=1020"},"modified":"2022-07-27T16:45:25","modified_gmt":"2022-07-27T08:45:25","slug":"pyppeteer%e9%80%9a%e7%94%a8%e6%a8%a1%e6%9d%bf","status":"publish","type":"post","link":"\/?p=1020","title":{"rendered":"pyppeteer\u901a\u7528\u6a21\u677f"},"content":{"rendered":"
\n
import asyncio\nimport time\nfrom loguru import logger\nlogger.add("out.log", rotation="100 MB", backtrace=True, diagnose=True)\n\nfrom subprocess import call\nfrom multiprocessing import Process\nfrom fake_useragent import UserAgent\nfrom pyppeteer import launch\nua = UserAgent()\nclass Spider:\n    def __init__(self):\n        self.url = "https:\/\/search.shopping.naver.com\/catalog\/19374887080?query=ADATA%20HD330%20USB3.1&NaPm=ct%3Dl4i53xi8%7Cci%3D7610a9621d487f37cd7a8e1cf287613ee02f1b25%7Ctr%3Dslsl%7Csn%3D95694%7Chk%3D45f91bfab8555ace9e5d7236982a0e7fffc115b0"\n        self.headers = {\n    "accept": "text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/avif,image\/webp,image\/apng,*\/*;q=0.8,application\/signed-exchange;v=b3;q=0.9",\n    "accept-encoding": "gzip, deflate, br",\n    "accept-language": "zh-CN,zh;q=0.9",\n    "cache-control": "no-cache",\n    "pragma": "no-cache",\n    "sec-fetch-dest": "document",\n    "sec-fetch-mode": "navigate",\n    "sec-fetch-site": "none",\n    "sec-fetch-user": "?1",\n    "upgrade-insecure-requests": "1",\n    "user-agent": ua.random\n}\n\n    async def get_html(self, i, num):\n        browser = await launch(\n            # headless=False,\n            headless=True,\n            dumpio=True,\n            # dumpio=False,\n            autoClose=True, devtools=False,\n            userDataDir=r"D:\\temporary",\n            args=['--disable-infobars',\n                  # '--no-sandbox',\n                  '--proxy-server=u6213.20.tn.16yun.cn:****',\n                  '--disable-setuid-sandbox',\n                  '--window-size=1366,850',\n\n                  ]\n        )\n        page = await browser.newPage()\n        # page = await browser.pages()\n        await page.setViewport({'width': 1366, 'height': 768})\n        await page.authenticate({\n            'username': '******',\n            'password': '******'\n        })\n\n        await page.setUserAgent('Mozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/94.0.4606.61 Safari\/537.36')\n        try:\n            await page.goto(self.url, {'timeout': 30 * 1000})\n            page_title = await page.title()\n            if "ADATA HD330 USB3.1" in page_title:\n                # await asyncio.sleep(2)\n                # await page.evaluate('window.scrollTo(0,document.body.800)')\n\n                # await page.evaluate('window.scrollTo(0,document.body.800)')\n                # selector = '#__next > div > div.style_container__3iYev > div.style_inner__1Eo2z > div.style_content_wrap__2VTVx > div.style_content__36DCX > div > div.summary_info_area__3XT5U > div.condition_area > table > tbody > tr:nth-child(6) > td.productByMall_mall_area__1oEU_ > div > a'\n                # \u70b9\u51fb\u7279\u5b9a\u5143\u7d20\n                selector = '#__next > div > div.style_container__3iYev > div.style_inner__1Eo2z > div.style_content_wrap__2VTVx > div.style_content__36DCX > div > div.summary_info_area__3XT5U > div.condition_area > table > tbody > tr:nth-child(4) > td.productByMall_mall_area__1oEU_ > div > a'\n                await page.click(selector)\n                await asyncio.sleep(2)\n                # print("\u8fdb\u7a0b{}\uff1a\u5237\u65b0{}\u6b21".format(num, i))\n                await page.evaluate('window.scrollTo(0,200)')\n                await page.evaluate('window.scrollTo(200,400)')\n                await page.evaluate('window.scrollTo(400,800)')\n                logger.info("\u8fdb\u7a0b{}\uff1a\u5237\u65b0{}\u6b21".format(num, i))\n                time.sleep(2)\n\n                await browser.close()\n\n        except Exception as e:\n            logger.error(e)\n        finally:\n            await browser.close()\n    def run(self, num):\n        i = 1\n        while True:\n            try:\n                asyncio.get_event_loop().run_until_complete(self.get_html(i, num))\n            except Exception as e:\n                logger.error(e)\n            i += 1\n\nif __name__ == '__main__':\n    # Spider().run(2)\n    for num in range(5):\n        p1 = Process(target=Spider().run, args=(str(num),))\n        p1.start()\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
import asyncio import time from loguru import logger logger.add(&quo   \u2026 …<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"ngg_post_thumbnail":0},"categories":[50],"tags":[54,55],"_links":{"self":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts\/1020"}],"collection":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1020"}],"version-history":[{"count":2,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/1020\/revisions"}],"predecessor-version":[{"id":1255,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/1020\/revisions\/1255"}],"wp:attachment":[{"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1020"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1020"},{"taxonomy":"post_tag","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1020"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}