How to Scrape Medium Articles and Author Stats with Python

Medium hosts millions of articles with engagement data. Scraping it helps analyze content strategies and benchmark authors. Hidden JSON Endpoints Append ?format=json to URLs. Response has anti-XSRF prefix ]}while(1);</x> : import requests , json , time from datetime import datetime class MediumScraper : def __init__ ( self ): self . s = requests . Session () self . s . headers . update ({ " User-Agent " : " Mozilla/5.0 " , " Accept " : " application/json " }) def _json ( self , text ): pfx = " ]}while(1);</x> " return json . loads ( text [ len ( pfx ):] if text . startswith ( pfx ) else text ) def profile ( self , user ): d = self . _json ( self . s . get ( f " https://medium.com/@ { user } ?format=json " ). text ) u = d . get ( " payload " ,{}). get ( " user " ,{}) return { " name " : u . get ( " name " ), " followers " : u . get ( " socialStats " ,{}). get ( " followerCount " , 0 )} def posts ( self , user , limit = 25 ): d = self . _json ( self . s . get ( f " https://medium.com/@ {