{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## **Web Scraping** at **dev.to**\n", "```\n", "Link Web Scraping :\n", "https://dev.to\n", "\n", "Code By Natasya \n", "medium.com/@liontin\n", "```" ] }, { "cell_type": "code", "execution_count": 505, "metadata": {}, "outputs": [], "source": [ "import requests\n", "import pandas as pd\n", "from urllib.parse import urlencode" ] }, { "cell_type": "code", "execution_count": 506, "metadata": {}, "outputs": [], "source": [ "key = input(' Tags Keyword :') # your tags search\n", "start = input('Number Star of Page') \n", "stop = input('Number Stop of Page') \n", "\n", "startpage = int(start) #validation number of start page\n", "stoppage = int(stop) #validation number of stop page\n", "\n", "devto = []\n", "\n", "url = 'https://dev.to/search/feed_content?'\n", "for page in range(startpage,stoppage+1):\n", " params = {\n", " 'page': page,\n", " 'tag': key,\n", " 'sort_by': 'hotness_score',\n", " 'sort_direction': 'desc',\n", " 'tag_names[]': key,\n", " 'approved': '',\n", " 'class_name': 'Article'\n", " }\n", "\n", " geturl = url+urlencode(params)\n", " r = requests.get(geturl).json()\n", "\n", " dev_article = r['result']\n", " for to in dev_article:\n", " title = to['title']\n", " author = to['user']['username']\n", " link = to['path']\n", " date = to['readable_publish_date']\n", " comments = to['comments_count']\n", " reactions = to['public_reactions_count']\n", " tags_list = to['tag_list']\n", " devto.append([author, title, reactions, comments, tags_list, date, str('https://dev.to%s' % link)])\n" ] }, { "cell_type": "code", "execution_count": 507, "metadata": {}, "outputs": [], "source": [ "dev_to = pd.DataFrame(devto,columns=['author', 'title','reactions','comments', 'tags', 'date', 'link'])" ] }, { "cell_type": "code", "execution_count": 508, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
authortitlereactionscommentstagsdatelink
0changachPython 101:Ultimate Guide To Python70[programming, python]Feb 13https://dev.to/changach/python-101ultimate-gui...
1brayan_kaiPython 101: Introduction to Modern Python101[python, programming, datascience, machinelear...Feb 13https://dev.to/brayan_kai/python-101-introduct...
2jit01Django Internationalization10[djangointernationalization, webpagetranslatio...Feb 9https://dev.to/epamindia/django-internationali...
3fridexWhy PyPI Doesn't Know Your Projects Dependenci...40[python, programming, opensource, security]Feb 13https://dev.to/fridex/why-pypi-doesnt-know-you...
4elliot_brenyasarfo_187495 Strange Things You Can Do With Python40[javascript, programming, beginners, python]Feb 9https://dev.to/elliot_brenyasarfo_18749/5-stra...
50xboltWeb3 authentication on Solana using Python⚡20[solana, web3, python, beginners]Feb 13https://dev.to/0xbolt/web3-authentication-on-s...
6adachimeIntroduction to modern python: Getting started...112[python, luxtechacademy, datascience, programm...Feb 13https://dev.to/adachime/introduction-to-python...
7wariarandirituPYTHON 10120[programming, python]Feb 13https://dev.to/wariarandiritu/python-101-3o57
8hannahmwendeIntroduction to modern python20[beginners, programming, python]Feb 13https://dev.to/hannahmwende/introduction-to-mo...
9baranselPython comment line50[python, beginners, tutorial]Feb 9https://dev.to/baransel/python-comment-line-1i5p
\n", "
" ], "text/plain": [ " author \\\n", "0 changach \n", "1 brayan_kai \n", "2 jit01 \n", "3 fridex \n", "4 elliot_brenyasarfo_18749 \n", "5 0xbolt \n", "6 adachime \n", "7 wariarandiritu \n", "8 hannahmwende \n", "9 baransel \n", "\n", " title reactions comments \\\n", "0 Python 101:Ultimate Guide To Python 7 0 \n", "1 Python 101: Introduction to Modern Python 10 1 \n", "2 Django Internationalization 1 0 \n", "3 Why PyPI Doesn't Know Your Projects Dependenci... 4 0 \n", "4 5 Strange Things You Can Do With Python 4 0 \n", "5 Web3 authentication on Solana using Python⚡ 2 0 \n", "6 Introduction to modern python: Getting started... 11 2 \n", "7 PYTHON 101 2 0 \n", "8 Introduction to modern python 2 0 \n", "9 Python comment line 5 0 \n", "\n", " tags date \\\n", "0 [programming, python] Feb 13 \n", "1 [python, programming, datascience, machinelear... Feb 13 \n", "2 [djangointernationalization, webpagetranslatio... Feb 9 \n", "3 [python, programming, opensource, security] Feb 13 \n", "4 [javascript, programming, beginners, python] Feb 9 \n", "5 [solana, web3, python, beginners] Feb 13 \n", "6 [python, luxtechacademy, datascience, programm... Feb 13 \n", "7 [programming, python] Feb 13 \n", "8 [beginners, programming, python] Feb 13 \n", "9 [python, beginners, tutorial] Feb 9 \n", "\n", " link \n", "0 https://dev.to/changach/python-101ultimate-gui... \n", "1 https://dev.to/brayan_kai/python-101-introduct... \n", "2 https://dev.to/epamindia/django-internationali... \n", "3 https://dev.to/fridex/why-pypi-doesnt-know-you... \n", "4 https://dev.to/elliot_brenyasarfo_18749/5-stra... \n", "5 https://dev.to/0xbolt/web3-authentication-on-s... \n", "6 https://dev.to/adachime/introduction-to-python... \n", "7 https://dev.to/wariarandiritu/python-101-3o57 \n", "8 https://dev.to/hannahmwende/introduction-to-mo... \n", "9 https://dev.to/baransel/python-comment-line-1i5p " ] }, "execution_count": 508, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dev_to.head(10)" ] } ], "metadata": { "interpreter": { "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" }, "kernelspec": { "display_name": "Python 3.6.8 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.8" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }