（Python）不能再使用re、请求和json从我的目标站点抓取数据

Question

我遇到了一个问题，我可以使用java路径从网站上刮取数据。我正试着从火箭联盟追踪器上搜刮。 这是我的密码：

import requests
import re
import json
import math

def rankGetter():

    trackerLink = 'https://rocketleague.tracker.network/rocket-league/profile/epic/DirectPanda/overview'

    # now we have the tracker link we're going to scrape the website
    # all the HTML of the site is now in result
    result = requests.get(trackerLink)

    # checker to make sure the user used the correct information
    if result.status_code == 400:
        print('profile not found')

    else:
        # Extract everything needed to render the current page. Data is stored as Json in the
        # JavaScript variable: window.__INITIAL_STATE__={"route":{"path":"\u0 ... }};
        json_string = re.search(r"window.__INITIAL_STATE__\s?=\s?(\{.*?\});", result.text).group(1)

        # convert text string to structured json data
        rocketleague = json.loads(json_string)

        # Save structured json data to a text file that helps you orient yourself and pick
        # the parts you are interested in.
        with open('rocketleague_json_data.txt', 'w') as outfile:
            outfile.write(json.dumps(rocketleague, indent=4, sort_keys=True))

错误是文档生成的文本不再具有我想要的级别

"stats": {
    "standardLeaderboardLeaders": {},
    "standardLeaderboards": [],
    "standardPlayers": {},
    "standardTitles": {}
},
**"stats-v2": {
    "segments": {},
    "standardProfileMatches": {},
    "standardProfileSummaries": {},
    "standardProfiles": {},
    "standardProfilesHistory": {},
    "standardSessions": {},
    "subscriptions": {}
},**
"titles": {
    "currentTitle": {
        "name": "Rocket League",
        "platforms": [

等级应该在stats-V2下，但是你现在可以看到它是空的。 发生了什么事，我该如何解决？我能得到一个星期的排名，但今天突然停止工作

Answer 1

数据似乎是从外部URL加载的：

import json
import requests


url = "https://api.tracker.gg/api/v2/rocket-league/standard/profile/epic/DirectPanda"

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
}

data = requests.get(url, headers=headers).json()
print(json.dumps(data, indent=4))

印刷品：

{
    "data": {
        "platformInfo": {
            "platformSlug": "epic",
            "platformUserId": null,
            "platformUserHandle": "DirectPanda",
            "platformUserIdentifier": "DirectPanda",
            "avatarUrl": null,
            "additionalParameters": null
        },
        "userInfo": {
            "userId": null,
            "isPremium": false,
            "isVerified": false,
            "isInfluencer": false,
            "isPartner": false,
            "countryCode": null,
            "customAvatarUrl": null,
            "customHeroUrl": null,
            "socialAccounts": [],
            "pageviews": 592,
            "isSuspicious": null
        },
        "metadata": {
            "lastUpdated": {
                "value": "2021-04-22T17:39:42.277-04:00",
                "displayValue": "2021-04-22T21:39:42.2770000+00:00"
            },
            "playerId": 16603481,
            "currentSeason": 17
        },
        "segments": [
            {
                "type": "overview",
                "attributes": {},
                "metadata": {
                    "name": "Lifetime"
                },
                "expiryDate": "0001-01-01T00:00:00+00:00",
                "stats": {
                    "wins": {
                        "rank": 30357,
                        "percentile": 98.3,
                        "displayName": "Wins",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 4985,
                        "displayValue": "4,985",
                        "displayType": "Number"
                    },
                    "goals": {
                        "rank": 23698,
                        "percentile": 98.7,
                        "displayName": "Goals",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 14363,
                        "displayValue": "14,363",
                        "displayType": "Number"
                    },
                    "mVPs": {
                        "rank": 35646,
                        "percentile": 98.0,
                        "displayName": "MVPs",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 2093,
                        "displayValue": "2,093",
                        "displayType": "Number"
                    },
                    "saves": {
                        "rank": 30864,
                        "percentile": 98.3,
                        "displayName": "Saves",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 9231,
                        "displayValue": "9,231",
                        "displayType": "Number"
                    },
                    "assists": {
                        "rank": 29228,
                        "percentile": 98.4,
                        "displayName": "Assists",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 4763,
                        "displayValue": "4,763",
                        "displayType": "Number"
                    },
                    "shots": {
                        "rank": 24596,
                        "percentile": 98.6,
                        "displayName": "Shots",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 29139,
                        "displayValue": "29,139",
                        "displayType": "Number"
                    },
                    "goalShotRatio": {
                        "rank": 1409320,
                        "percentile": 15.0,
                        "displayName": "Goal Shot Ratio",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 49.29132777377398,
                        "displayValue": "49.3",
                        "displayType": "NumberPrecision1"
                    },
                    "score": {
                        "rank": 28260,
                        "percentile": 98.4,
                        "displayName": "TRN Score",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {},
                        "value": 2398222.83,
                        "displayValue": "2,398,222.8",
                        "displayType": "NumberPrecision1"
                    },
                    "seasonRewardLevel": {
                        "rank": null,
                        "percentile": 85.0,
                        "displayName": "Season Reward Level",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-13.png",
                            "rankName": "Diamond"
                        },
                        "value": 5,
                        "displayValue": "5",
                        "displayType": "Number"
                    },
                    "seasonRewardWins": {
                        "rank": null,
                        "percentile": 95.8,
                        "displayName": "Season Reward Wins",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {},
                        "value": 9,
                        "displayValue": "9",
                        "displayType": "Number"
                    }
                }
            },
            {
                "type": "playlist",
                "attributes": {
                    "playlistId": 0,
                    "season": 17
                },
                "metadata": {
                    "name": "Un-Ranked"
                },
                "expiryDate": "0001-01-01T00:00:00+00:00",
                "stats": {
                    "tier": {
                        "rank": null,
                        "percentile": null,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-0.png",
                            "name": "Unranked"
                        },
                        "value": 0,
                        "displayValue": "0",
                        "displayType": "Number"
                    },
                    "division": {
                        "rank": null,
                        "percentile": null,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "name": "Division I"
                        },
                        "value": 0,
                        "displayValue": "0",
                        "displayType": "Number"
                    },
                    "matchesPlayed": {
                        "rank": null,
                        "percentile": null,
                        "displayName": "Matches",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 0,
                        "displayValue": "0",
                        "displayType": "Number"
                    },
                    "winStreak": {
                        "rank": null,
                        "percentile": null,
                        "displayName": "WinStreak",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {
                            "type": "win"
                        },
                        "value": 0,
                        "displayValue": "0",
                        "displayType": "Number"
                    },
                    "rating": {
                        "rank": 215152,
                        "percentile": 90.0,
                        "displayName": "Rating",
                        "displayCategory": "Skill",
                        "category": "skill",
                        "metadata": {},
                        "value": 1597,
                        "displayValue": "1,597",
                        "displayType": "Number"
                    }
                }
            },
            {
                "type": "playlist",
                "attributes": {
                    "playlistId": 10,
                    "season": 17
                },
                "metadata": {
                    "name": "Ranked Duel 1v1"
                },
                "expiryDate": "0001-01-01T00:00:00+00:00",
                "stats": {
                    "tier": {
                        "rank": null,
                        "percentile": 98.2,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
                            "name": "Champion I"
                        },
                        "value": 16,
                        "displayValue": "16",
                        "displayType": "Number"
                    },
                    "division": {
                        "rank": null,
                        "percentile": 88.0,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "deltaDown": 13,
                            "deltaUp": 6,
                            "name": "Division III"
                        },
                        "value": 2,
                        "displayValue": "2",
                        "displayType": "Number"
                    },
                    "matchesPlayed": {
                        "rank": null,
                        "percentile": 57.0,
                        "displayName": "Matches",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 2,
                        "displayValue": "2",
                        "displayType": "Number"
                    },
                    "winStreak": {
                        "rank": null,
                        "percentile": 60.0,
                        "displayName": "WinStreak",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {
                            "type": "win"
                        },
                        "value": 1,
                        "displayValue": "1",
                        "displayType": "Number"
                    },
                    "rating": {
                        "rank": 101541,
                        "percentile": 96.1,
                        "displayName": "Rating",
                        "displayCategory": "Skill",
                        "category": "skill",
                        "metadata": {},
                        "value": 1031,
                        "displayValue": "1,031",
                        "displayType": "Number"
                    }
                }
            },
            {
                "type": "playlist",
                "attributes": {
                    "playlistId": 11,
                    "season": 17
                },
                "metadata": {
                    "name": "Ranked Doubles 2v2"
                },
                "expiryDate": "0001-01-01T00:00:00+00:00",
                "stats": {
                    "tier": {
                        "rank": null,
                        "percentile": 87.0,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
                            "name": "Champion I"
                        },
                        "value": 16,
                        "displayValue": "16",
                        "displayType": "Number"
                    },
                    "division": {
                        "rank": null,
                        "percentile": 90.0,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "deltaDown": 15,
                            "deltaUp": 3,
                            "name": "Division IV"
                        },
                        "value": 3,
                        "displayValue": "3",
                        "displayType": "Number"
                    },
                    "matchesPlayed": {
                        "rank": null,
                        "percentile": 80.0,
                        "displayName": "Matches",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 40,
                        "displayValue": "40",
                        "displayType": "Number"
                    },
                    "winStreak": {
                        "rank": null,
                        "percentile": 34.0,
                        "displayName": "WinStreak",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {
                            "type": "loss"
                        },
                        "value": 1,
                        "displayValue": "-1",
                        "displayType": "Number"
                    },
                    "rating": {
                        "rank": 311789,
                        "percentile": 89.0,
                        "displayName": "Rating",
                        "displayCategory": "Skill",
                        "category": "skill",
                        "metadata": {},
                        "value": 1177,
                        "displayValue": "1,177",
                        "displayType": "Number"
                    }
                }
            },
            {
                "type": "playlist",
                "attributes": {
                    "playlistId": 13,
                    "season": 17
                },
                "metadata": {
                    "name": "Ranked Standard 3v3"
                },
                "expiryDate": "0001-01-01T00:00:00+00:00",
                "stats": {
                    "tier": {
                        "rank": null,
                        "percentile": 96.0,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-17.png",
                            "name": "Champion II"
                        },
                        "value": 17,
                        "displayValue": "17",
                        "displayType": "Number"
                    },
                    "division": {
                        "rank": null,
                        "percentile": 79.0,
                        "displayName": "Matches",
                        "displayCategory": "General",
                        "category": "general",
                        "metadata": {
                            "deltaDown": 7,
                            "deltaUp": 27,
                            "name": "Division III"
                        },
                        "value": 2,
                        "displayValue": "2",
                        "displayType": "Number"
                    },
                    "matchesPlayed": {
                        "rank": null,
                        "percentile": 97.8,
                        "displayName": "Matches",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {},
                        "value": 95,
                        "displayValue": "95",
                        "displayType": "Number"
                    },
                    "winStreak": {
                        "rank": null,
                        "percentile": 16.0,
                        "displayName": "WinStreak",
                        "displayCategory": "Performance",
                        "category": "performance",
                        "metadata": {
                            "type": "loss"
                        },
                        "value": 2,
                        "displayValue": "-2",
                        "displayType": "Number"
                    },
                    "rating": {
                        "rank": 122500,
                        "percentile": 95.8,
                        "displayName": "Rating",
                        "displayCategory": "Skill",
                        "category": "skill",
                        "metadata": {},
                        "value": 1255,
                        "displayValue": "1,255",
                        "displayType": "Number"
                    }
                }
            },


...