{"version":"1.0","provider_name":"Fortune Insight","provider_url":"https:\/\/prime.fortuneinsight.com\/web","author_name":"\u9322\u741b","author_url":"https:\/\/prime.fortuneinsight.com\/web\/posts\/author\/chinshun","title":"\u300a\u5de5\u5177\u904b\u7528\u300b\u5982\u4f55\u5feb\u901f\u5b78\u7fd2Web Scraping\uff1f - Fortune Insight","type":"rich","width":600,"height":338,"html":"<blockquote class=\"wp-embedded-content\"><a href=\"https:\/\/prime.fortuneinsight.com\/web\/posts\/559304\">\u300a\u5de5\u5177\u904b\u7528\u300b\u5982\u4f55\u5feb\u901f\u5b78\u7fd2Web Scraping\uff1f<\/a><\/blockquote>\n<script type='text\/javascript'>\n<!--\/\/--><![CDATA[\/\/><!--\n\t\t!function(c,d){\"use strict\";var e=!1,n=!1;if(d.querySelector)if(c.addEventListener)e=!0;if(c.wp=c.wp||{},!c.wp.receiveEmbedMessage)if(c.wp.receiveEmbedMessage=function(e){var t=e.data;if(t)if(t.secret||t.message||t.value)if(!\/[^a-zA-Z0-9]\/.test(t.secret)){for(var r,a,i,s=d.querySelectorAll('iframe[data-secret=\"'+t.secret+'\"]'),n=d.querySelectorAll('blockquote[data-secret=\"'+t.secret+'\"]'),o=0;o<n.length;o++)n[o].style.display=\"none\";for(o=0;o<s.length;o++)if(r=s[o],e.source===r.contentWindow){if(r.removeAttribute(\"style\"),\"height\"===t.message){if(1e3<(i=parseInt(t.value,10)))i=1e3;else if(~~i<200)i=200;r.height=i}if(\"link\"===t.message)if(a=d.createElement(\"a\"),i=d.createElement(\"a\"),a.href=r.getAttribute(\"src\"),i.href=t.value,i.host===a.host)if(d.activeElement===r)c.top.location.href=t.value}}},e)c.addEventListener(\"message\",c.wp.receiveEmbedMessage,!1),d.addEventListener(\"DOMContentLoaded\",t,!1),c.addEventListener(\"load\",t,!1);function t(){if(!n){n=!0;for(var e,t,r=-1!==navigator.appVersion.indexOf(\"MSIE 10\"),a=!!navigator.userAgent.match(\/Trident.*rv:11\\.\/),i=d.querySelectorAll(\"iframe.wp-embedded-content\"),s=0;s<i.length;s++){if(!(e=i[s]).getAttribute(\"data-secret\"))t=Math.random().toString(36).substr(2,10),e.src+=\"#?secret=\"+t,e.setAttribute(\"data-secret\",t);if(r||a)(t=e.cloneNode(!0)).removeAttribute(\"security\"),e.parentNode.replaceChild(t,e)}}}}(window,document);\n\/\/--><!]]>\n<\/script><iframe sandbox=\"allow-scripts\" security=\"restricted\" src=\"https:\/\/prime.fortuneinsight.com\/web\/posts\/559304\/embed\" width=\"600\" height=\"338\" title=\"&#8220;\u300a\u5de5\u5177\u904b\u7528\u300b\u5982\u4f55\u5feb\u901f\u5b78\u7fd2Web Scraping\uff1f&#8221; &#8212; Fortune Insight\" frameborder=\"0\" marginwidth=\"0\" marginheight=\"0\" scrolling=\"no\" class=\"wp-embedded-content\"><\/iframe>","thumbnail_url":"https:\/\/prime.fortuneinsight.com\/web\/wp-content\/uploads\/2021\/07\/selenium_logo.jpg","thumbnail_width":284,"thumbnail_height":305,"description":"\u6709\u7db2\u53cb\u554f\u61c9\u8a72\u5982\u4f55\u5b78\u7fd2\u7db2\u9801\u64ae\u53d6\u00a0\uff08web scraping\uff09\uff0c\u4ee5\u53ca\u61c9\u5426\u5c07\u9019\u985e\u5de5\u4f5coutsource\u51fa\u53bb\u627e\u4eba\u505a\u3002\u7531\u65bc\u9019\u500b\u984c\u76ee\u5f88\u95dc\u4e4e\u91cf\u5316\u5206\u6790\u7684\u6578\u64da\u4f86\u6e90\uff0c\u60f3\u505a\u4e00\u500b\u8f03\u8a73\u7d30\u7684\u89e3\u7b54\u3002 \u5148\u5b9a\u7fa9web scraping\u3002\u9019\u500b\u5b57\u7684\u610f\u601d\u662f\u5728\u4e0d\u540c\u7684\u7db2\u9801\u4e0a\u6536\u96c6\u8cc7\u6599\uff0c\u4e26\u4ee5\u7a0b\u5f0f\u4ee3\u66ff\u4eba\u624b\u9032\u884c\u64cd\u4f5c\u3002\u9032\u884cweb scraping\u7684\u7a0b\u5f0f\uff0c\u7a31\u70baweb scraper\uff0c\u53ef\u4ee5\u64ae\u53d6\u4f60\u8a8d\u70ba\u6709\u7528\u7684\u6578\u64da\uff0c\u4f8b\u5982\u6e2f\u4ea4\u6240\u7db2\u9801\u7684\u671f\u8ca8\u3001\u671f\u6b0a\u6578\u64da\uff0c\u6216\u4e0d\u540c\u52a0\u5bc6\u8ca8\u5e63\u4ea4\u6613\u6240\u5c55\u793a\u7684non price data\u3002 \u5982\u679c\u81ea\u5df1\u6c92\u6709\u7de8\u7a0b\u80cc\u666f\uff0c\u627e\u66ff\u5de5\u7de8\u5bebweb scraper\u662f\u5426\u53ef\u53d6\uff1f\u5176\u5be6\u8996\u4e4e\u66ff\u5de5\u7684\u8cea\u7d20\u3002\u7531\u65bc\u7de8\u5bebweb scraper\u4e00\u822c\u90fd\u5e0c\u671b\u62ff\u53d6\u6bd4\u8f03customised\u7684\u6578\u64da\uff0c\u5c31\u7b97\u80fd\u5728\u52de\u5de5\u5e02\u5834\u627e\u5230\u61c2\u5f97\u7de8\u5bebweb scraper\u7684programmer\uff0c\u4e5f\u672a\u5fc5\u53ef\u4ee5100%\u6eff\u8db3\u81ea\u5df1\u7684\u8981\u6c42\u3002 \u66f4\u91cd\u8981\u7684\u662f\uff0c\u5982\u679c\u6578\u64da\u4f86\u6e90\u7684\u7db2\u9801\u6709\u683c\u5f0f\u4e0a\u7684\u8f49\u8b8a\uff0c\u4ee4\u539f\u672c\u7684web scraper\u4e0d\u80fd\u904b\u4f5c\uff0c\u81ea\u5df1\u61c2\u5f97web scraping\u6280\u5de7\u7684\u8a71\uff0c\u5c31\u53ef\u4ee5\u66f4\u5feb\u9032\u884c\u76f8\u61c9\u8abf\u7bc0\uff0c\u4e0d\u6703\u932f\u5931\u4efb\u4f55\u6642\u9593\u7684\u6578\u64da\u9ede\u3002 \u90a3\u9ebc\uff0cweb scraping\u96e3\u5b78\u55ce\uff1f\u61c9\u8a72\u5982\u4f55\u5b78\u7fd2web scraping\uff1f\u500b\u4eba\u7d93\u9a57\u800c\u8a00\uff0c\u5047\u5982\u6c92\u6709\u4ebaguide\uff0c\u81ea\u5b78\u7684\u78ba\u6703\u6bd4\u8f03\u6162\u3002\u4e0d\u904e\uff0c\u5982\u679c\u6709\u4e00\u4e9b\u57fa\u672c\u65b9\u5411\uff0c\u6216\u6709\u5c0dweb scraping\u6846\u67b6\u7684\u8a8d\u8b58\uff0c\u81ea\u5b78\u5176\u5be6\u4e5f\u53ef\u4ee5\u5f88\u5feb\u3002 \u7b2c\u4e00\u500b\u8981\u7406\u89e3\u7684\u6982\u5ff5\uff0c\u662fweb scraping\u672c\u8cea\u4e0a\u5176\u5be6\u8981\u5206\u6210\u5169\u6b65\uff1ascraping\u53ca\u5c0d\u7db2\u9801\u9032\u884c\u89e3\u6790\uff08parsing\uff09\u3002\u56b4\u683c\u4f86\u8aaa\uff0cscraping\u6307\u4e0b\u8f09\u7db2\u9801\u5167\u5bb9\uff0cparsing\u5247\u6307\u5f9e\u5df2\u4e0b\u8f09\u7684\u7db2\u9801\u5167\u5bb9\u4e2d\u62bd\u53d6\u6709\u7528\u6578\u64da\u3002\u4ee5\u4e0a\u63d0\u53ca\u7684\u7db2\u9801\u5167\u5bb9\uff0c\u4e00\u822c\u662f\u6307html\u6a94\u6848\uff0c\u4ea6\u5373\u627f\u8f09\u7740\u6574\u500b\u7db2\u9801\u7684\u7d50\u69cb\u53ca\u6578\u64da\u7684\u6a94\u6848\u3002 \u6240\u4ee5\uff0c\u8981\u505ascraping\uff08\u4e0b\u8f09\uff09\u5176\u5be6\u4e0d\u96e3\uff0c\u66f4\u8003\u6280\u5de7\u7684\u53cd\u800c\u662fparsing\u3002\u9019\u88cf\u5c31\u8981\u4ecb\u7d39\u7b2c\u4e8c\u500b\u6982\u5ff5\uff1a\u9078\u64c7\u5668\uff08selector\uff09\u3002\u5f9e\u4e00\u500b\u7db2\u9801\u6a94\u6848\u4e2d\u62bd\u53d6\u6709\u7528\u7684\u8cc7\u6599\uff0c\u5c31\u5fc5\u8981\u77e5\u9053\u8cc7\u6599\u7684\u4f4d\u7f6e\uff0cselector\u5c31\u662f\u8868\u9054\u9019\u7a2e\u4f4d\u7f6e\u7684\u65b9\u5f0f\u3002\u884c\u5167\u6709\u5169\u5927selector\u53ef\u4f9b\u9078\u64c7\uff0c\u5206\u5225\u70baxpath\u53caCSS Selector\u3002\u500b\u4eba\u8a8d\u70ba\u5b78\u61c2xpath\u5df2\u7d93\u7dbd\u7dbd\u6709\u9918\u3002\u800c\u4e14\uff0cxpath\u7684\u8a9e\u6cd5\u53ca\u908f\u8f2f\u90fd\u5f88\u50cfWindows\u7684\u6a94\u6848\u8def\u5f91\uff08file path\uff09\uff0c\u521d\u5b78\u8005\u4e5f\u6703\u5f88\u6613\u4e0a\u624b\u3002 \u7136\u5f8c\uff0c\u5c31\u8981\u7406\u89e3\u5c0d\u7db2\u9801\u9032\u884cscraping\uff08\u4e0b\u8f09\uff09\u7684\u6846\u67b6\u7684\u5206\u5225\u3002\u6bd4\u8f03\u5e38\u7528\u7684\u6709\u5169\u7a2e\u6846\u67b6\uff0c\u5206\u5225\u61c9\u4ed8\u4e0d\u540c\u985e\u578b\u7684\u7db2\u9801\u3002\u5982\u679c\u7db2\u9801\u5167\u5bb9\u6bd4\u8f03\u7c21\u55ae\uff0c\u6c92\u6709\u5927\u91cf\u7684JavaScript\u4ee4\u7db2\u9801\u5167\u5bb9\u4e0d\u65b7\u66f4\u65b0\uff08\u4f8b\u5982\u6e2f\u4ea4\u6240\u7684\u90e8\u5206\u7db2\u9801\uff09\uff0c\u4e00\u822c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528get request\u4e0b\u8f09\u6574\u500b\u7db2\u9801\u7684\u5167\u5bb9\uff0c\u5728python\u900f\u904erequests package\u5c31\u53ef\u4ee5\u76f4\u63a5\u57f7\u884c\u3002 \u4f46\u662f\uff0c\u5982\u679c\u7db2\u9801\u6709\u5927\u91cf\u7684\u5167\u5bb9\u66f4\u65b0\uff08\u4f8b\u5982\u67d0\u4e9b\u52a0\u5bc6\u8ca8\u5e63\u4ea4\u6613\u6240\u7684\u7db2\u9801\uff09\uff0c\u4f7f\u7528get request\u5c31\u4e0d\u80fd\u5b8c\u6574\u5730\u5c07\u6240\u6709\u6578\u64da\u4e0b\u8f09\uff0c\u56e0\u6b64\u5c31\u8981\u4f7f\u7528\u53e6\u5916\u4e00\u500bscraping\u6846\u67b6\uff1a\u9023\u7d50\u700f\u89bd\u5668\uff08connected browser\uff09\u3002\u9019\u500b\u6846\u67b6\u7684\u6982\u5ff5\u5176\u5be6\u76f8\u7576\u7c21\u55ae\uff0c\u53ea\u662f\u4ee5\u7a0b\u5f0f\u4ee3\u66ff\u4eba\u624b\u4e0d\u65b7\u700f\u89bd\u7db2\u9801\uff0c\u4f8b\u5982\u586b\u5beb\u767b\u5165\u8cc7\u6599\u3001\u63c0\u9078\u8cc7\u6599\u7be9\u9078\u5668\u3001\u6309\u5230\u4e0b\u4e00\u9801\u7b49\u7b49\u3002\u9019\u6a23\u5c31\u53ef\u4ee5\u5c07\u81ea\u5df1\u60f3\u8981\u7684\u6578\u64da\u5168\u90e8\u986f\u793a\u51fa\u4f86\uff0c\u7136\u5f8c\u9032\u884c\u4e0b\u8f09\u3002\u5728python\u4e2d\uff0cSelenium\u5c31\u662f\u57f7\u884c\u9019\u500b\u6846\u67b6\u7684\u6700\u4e3b\u6d41package\u3002 \u5207\u8a18\uff0c\u4e0d\u8ad6\u4f7f\u7528\u54ea\u4e00\u7a2escraping\u65b9\u6cd5\uff0c\u4e0b\u8f09\u7684\u983b\u7387\u4e5f\u4e0d\u80fd\u592a\u9ad8\uff0c\u4e00\u822c\u800c\u8a00\u8981\u76f8\u969410\u79d2\u6216\u4ee5\u4e0a\u624d\u53ef\u4ee5\u4e0b\u8f09\u53e6\u4e00\u500b\u7db2\u9801\uff0c\u5426\u5247\u4f60\u7684IP address\u6703\u88ab\u8996\u70ba\u5c0d\u7db2\u9801\u9032\u884c\u653b\u64ca\uff0c\u5f88\u5927\u6a5f\u6703\u6703\u88ab\u7981\u6b62\u700f\u89bd\u7db2\u9801\u3002 \u6700\u5f8c\uff0c\u5c31\u662f\u5982\u4f55\u5132\u5b58\u6578\u64da\u7684\u554f\u984c\u3002\u6578\u64da\u91cf\u5c11\u7684\u8a71\uff0c\u53ef\u4ee5\u5c07\u4e0b\u8f09\u7684\u7db2\u9801\u5167\u5bb9\u653e\u5230\u96fb\u8166\u7684\u786c\u76e4\uff08disk\uff09\u4e2d\uff0c\u4ea6\u5373\u5c07\u6578\u64da\u4ee5\u6a94\u6848\u7684\u5f62\u5f0f\u653e\u5230\u96fb\u8166\u5167\u3002\u4f46\u662f\uff0c\u6578\u64da\u91cf\u592a\u591a\u7684\u8a71\uff0c\u9019\u500b\u65b9\u6cd5\u5c31\u6703\u8b8a\u5f97\u76f8\u7576\u6c92\u6709\u6548\u7387\uff0c\u56e0\u70ba\u4e0d\u8ad6\u5132\u5b58\u3001\u641c\u7d22\u3001\u4ee5\u81f3parsing\u7684\u904b\u884c\u6642\u9593\u90fd\u6703\u5f88\u9577\u3002 \u56e0\u6b64\uff0c\u5982\u679c\u8981\u8655\u7406\u5927\u6578\u64da\uff0c\u5c31\u53ef\u80fd\u8981\u4f7f\u7528\u53e6\u5916\u4e00\u7a2e\u5f62\u5f0f\u7684\u6578\u64da\u5eab\u5c07\u7db2\u9801\u6578\u64da\u5132\u5b58\u8d77\u4f86\uff0c\u4f8b\u5982SQL\u6216mongoDB\u3002\u7576\u7136\uff0c\u9019\u500b\u65b9\u6cd5\u7684\u64cd\u4f5c\u76f8\u5c0d\u8907\u96dc\uff0c\u5c0d\u7de8\u7a0b\u6709\u4e00\u5b9a\u8981\u6c42\u3002 \u7e3d\u62ec\u800c\u8a00\uff0c\u8981\u5165\u624bweb scraping\u7684\u8a71\uff0c\u53ef\u4ee5\u8a18\u4f4f\u4ee5\u4e0b\u7684\u95dc\u9375\u5b57\uff1axpath\u3001requests\u3001\u4ee5\u53caSelenium\u3002Google\u4e00\u4e0b\u5c31\u6703\u767c\u73fe\u6709\u4e0d\u5c11\u53ef\u9760\u7684\u81ea\u5b78\u6e20\u9053\uff0c\u521d\u5b78\u8005\u4e5f\u80fd\u5f88\u5feb\u4e0a\u624b\u3002 \u6709\u4ec0\u9ebc\u554f\u984c\u6b61\u8fce\u518d\u554f\uff0c\u5e0c\u671b\u5e6b\u5230\u5927\u5bb6\u3002 \u9322\u741b"}