Instagram 爬虫¶

pip install instagram-scraper

instagram-scraper <username> #Scraping is done anonymously. 
instagram-scraper <username> -u <your username> -p <your password> #You log into your account and you make the scraping from there. Can be useful for private accounts. 

instagram-scraper <hashtag> --tag #Important! Don't include # when you enter the hashtag.

instagram-scraper <username> -u <your username> -p <your password> -t story --latest #For stories
instagram-scraper <username> --latest #Publication feeds

import os.path, time

directory = "psg"
file_list = []
for i in os.listdir(directory): #Here we list files in a folder and get the creation date
    a = os.stat(os.path.join(directory,i))
    file_list.append([i,str(time.ctime(os.path.getmtime(directory + "/" + i)))[0:10] + " " + str(time.ctime(os.path.getmtime(directory + "/" + i)))[-4::]]) #[file,creationtime]

today = str(time.ctime())[0:10] + " " + str(time.ctime())[-4::]

for x in file_list: #We compare today´s date and the creation date and if they do not match we delete the file
    if x[1] != today:
        os.remove(<path to PSG directory>"/psg/" + str(x[0]))
        print("removed")

os.rename("psg", "psg" + " " + today) #We change the name of the folder including the date

import json
with open(<your_JSON_file>) as json_file:     
    data = json.load(json_file) #JSON file is loaded.

listowners = []
for x in range (len(data["GraphImages"])):
    listowners.append(data["GraphImages"][x]["owner"]["id"]) #We iterate through the JSON file and we store the owner IDs.

listowners = list( dict.fromkeys(listowners)) #We remove duplicate owner IDs just in case.

from bs4 import BeautifulSoup
import cloudscraper

listaccounts = []
for x in listowners: 
    headers = {'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 Instagram 12.0.0.16.90 (iPhone9,4; iOS 10_3_3; en_US; en-US; scale=2.61; gamut=wide; 1080x1920)'} #We need to use Instagram UA to be able to access to this URL.
    parser = 'html.parser' 
    scraper = cloudscraper.create_scraper() 
    html = scraper.get("https://i.instagram.com/api/v1/users/"+str(x)+"/info/", headers = headers) #We request the URL
    soup = BeautifulSoup(html.text, parser).get_text() #We parse the HTML code which is returned from the request.
    jsondata = json.loads(soup) #We transform the HTML into Json.
    listaccounts.append(jsondata["user"]["username"]) #We append the usernames.

import re

html = scraper.get("https://www.instagram.com/<username>/?__a=1", headers = headers) #We request this URL which returns the Instagram profile with structured data
soup = BeautifulSoup(html.text, parser).get_text()
jsondata = json.loads(soup)

#Keys for each of the variables mentioned above
biography = jsondata["graphql"]["user"]["biography"]
externalurl = jsondata["graphql"]["user"]["external_url"]
followers = jsondata["graphql"]["user"]["edge_followed_by"]["count"]
following = jsondata["graphql"]["user"]["edge_follow"]["count"]
businessacount = jsondata["graphql"]["user"]["is_business_account"]
category = jsondata["graphql"]["user"]["overall_category_name"]
emails = re.findall(r"[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.com", soup, re.I)

import json
import pandas as pd

with open('psg.json') as json_file:     
    data = json.load(json_file) #We load the Json file

listposts = []
for x in range (len(data["GraphImages"])): #We iterate through the Json file and we get the variables through the keys
    try:
        media = data["GraphImages"][x]["display_url"]
        likes = data["GraphImages"][x]["edge_media_preview_like"]["count"]
        comments = data["GraphImages"][x]["edge_media_to_comment"]["count"]
        listposts.append([media,likes,comments])
    except:
        continue

df = pd.DataFrame(listposts, columns=["Media","Likes","Comments"])
df.to_csv('TestJason.csv', index=False) #We store the list in an Excel file by using Pandas