python post请求中的Graphql查询错误

Question

我在用python发送post请求时遇到了问题，我想刮取这个url的数据https://www.wlw.de/de/suche/page/3?employeeCounts=200%2B_50-199&q=verpackungen

当您单击此页面底部的“加载其他”按钮时，将在“网络”选项卡中发出post请求 我想要python中的post请求，但它给了我一个错误

这是我的密码

import requests

headers = {
    'authority':'www.wlw.de',
    'method':'POST',
    'path':'/unified_search_backend/graphql',
    'scheme':'https',
    'accept':'application/json, text/plain, */*',
    'accept-encoding':'gzip, deflate, br',
    'accept-language':'en-US,en;q=0.9',
    'content-type':'application/graphql',
    'cookie':'ab_testing_variants=%7B%22adEngine_2021_01%22%3A1%2C%22contact_form_page%22%3A1%2C%2242_mobileBanner_v1%22%3A0%2C%2239_shortProfile_v1%22%3A0%7D; wlw_client_id=rBEAB2BTd2qrtQDuA0v3Ag==; CookieConsent={stamp:%27mZs4JjRbQH+Y5duvSvOezU6hAh/OkfCEvrvzPwHc5dcNcsUiNE7uAQ==%27%2Cnecessary:true%2Cpreferences:true%2Cstatistics:true%2Cmarketing:true%2Cver:4%2Cutc:1616082805877%2Cregion:%27pk%27}; _ga=GA1.2.380141126.1616082808; _hjid=5740f01e-7ac9-4eb6-b4d4-1d6d8eac5a9b; aam_uuid=41781454427930149023915598636228190638; hubspotutk=a2547e1e94388ab650b0b636bdda1f38; _fbp=fb.1.1616082818609.443427339; i18n_redirected=en; ufs_session_id=4b8697cc5b2056f4f67288f7bde39ad2; _gid=GA1.2.1732485982.1619807573; AMCVS_41833DF75A550B4B0A495DA6%40AdobeOrg=1; AMCV_41833DF75A550B4B0A495DA6%40AdobeOrg=359503849%7CMCIDTS%7C18748%7CMCMID%7C41166180535232718843931056836287665940%7CMCAAMLH-1620412373%7C3%7CMCAAMB-1620412373%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1619814773s%7CNONE%7CvVersion%7C5.0.1; _hjIncludedInSessionSample=0; _hjTLDTest=1; _hjAbsoluteSessionInProgress=0; axd=4259004108571865363; __hstc=80469576.a2547e1e94388ab650b0b636bdda1f38.1616082816445.1616403237215.1619807575061.3; __hssrc=1; ab_testing_traffic_split_group=explore_b; _gaexp=GAX1.2.8oaBCv1OQnO_Jo4FzYdtxQ.18823.3!yUGcPeeNQI64hKsFDyWxPg.18823.0!vEscdyRLSkWhD-3KR-R8Tw.18823.0!LnHlGkhVTKW8ENYICsLX3w.18832.0!9CUCt3yLQ4eLUOs13vwPsg.18839.0; _gcl_au=1.1.1033109662.1619807603; ufs_tracking_data=%7B%22searchType%22%3A%22ufs%22%2C%22verticalSlug%22%3A%22packaging%22%2C%22concept%22%3A%22Verpackungen%22%2C%22companyPosition%22%3A61%2C%22spaceType%22%3A%22relevance%22%7D; ab_testing_variants_v2=%7B%22adEngine_2021_01%22%3A%7B%22name%22%3A%22ad_engine%22%2C%22dimension%22%3A%22wlwTestGroup%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412717%7D%2C%2239_shortProfile_v1%22%3A%7B%22name%22%3A%22none%22%2C%22dimension%22%3A%22wlwTestGroup3%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412717%7D%2C%2248_connectInSerp%22%3A%7B%22name%22%3A%22connect_funnel%22%2C%22dimension%22%3A%22wlwTestGroup4%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412717%7D%2C%2249_aaTest%22%3A%7B%22name%22%3A%2249_b%22%2C%22dimension%22%3A%22wlwTestGroup8%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412707%7D%2C%2251_exit_intent%22%3A%7B%22name%22%3A%22connect%22%2C%22dimension%22%3A%22wlwTestGroup9%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412707%7D%2C%2250_sidebar%22%3A%7B%22name%22%3A%22new_sidebar%22%2C%22dimension%22%3A%22wlwTestGroup11%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412707%7D%2C%2252_business_section%22%3A%7B%22name%22%3A%22new_business_section%22%2C%22dimension%22%3A%22wlwTestGroup10%22%2C%22active%22%3Atrue%2C%22expiry%22%3A1620412707%7D%7D; wlw_search_term=verpackungen; category_id=93737; __hssc=80469576.6.1619807575061; _gat_UA-38607859-4=1',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
    'x-application-id':'website_wlw',
    'x-original-url':'https://www.wlw.de/de/suche?employeeCounts=200%2B_50-199&q=verpackungen',
    'x-referer':'https://www.wlw.de/de/suche/page/2?employeeCounts=200%2B_50-199&q=verpackungen'
}

payload = '{"query":"query(\n        $ufsSessionId: String,\n        $searchMethod: String,\n        $q: String,\n        $conceptSlug: String,\n        $citySlug: String,\n        $filters: [FilterInput!],\n        $page: Int,\n        $sort: String,\n        $tracking: Boolean,\n        $disableCityExtraction: Boolean,\n        $cityExtractionRadius: String\n      ) {\n      search(\n        ufsSessionId: $ufsSessionId,\n        searchMethod: $searchMethod,\n        q: $q,\n        conceptSlug: $conceptSlug,\n        citySlug: $citySlug,\n        filters: $filters,\n        page: $page,\n        sort: $sort,\n        tracking: $tracking,\n        disableCityExtraction: $disableCityExtraction,\n        cityExtractionRadius: $cityExtractionRadius\n      ) {\n        ufsSessionId\n        paging {\n          total, currentPage, totalPages\n        }\n        companies {\n          id, uuid, name, highlightedName, description, highlightedDescription, secondaryHighlightedDescription,\n          homepage, zipcode, city, countryCode, certificates, logoPath, profilePagePath, distance, email,\n          debugData, employeeCount, foundingYear, acronym, customerId, isCustomer, fpaket, spaceType\n          pictures {\n            caption, path, position, sizes\n          }\n          logo {\n            sizes\n          }\n          filterResults {\n            key, title, value, matches\n          }\n          matchingCompanyCategory {\n            id, kpaket, toprankingPosition\n          }\n        }\n        filters {\n          key, name, static, title, type, range {\n            max, min, unit, disabled\n          }\n          values {\n            count, name, title\n          }\n          location {\n            kind\n            defaultRadiuses\n            availableRadiuses {\n              name, count\n            }\n          }\n        }\n        cities {\n          slug, title, count\n        }\n        suggestions {\n          text, score\n        }\n        extractedCity {\n          name, radius\n        }\n        toprankingPositions {\n          companyId, position\n        }\n        companyCategory {\n          id, name\n        }\n        concept {\n          title\n        }\n        recommendations {\n          text, score, slug\n        }\n        canonicalConceptSlug\n        canonicalCompanyCategorySlug\n        conceptSlugTitle\n        companyCategorySlugTitle\n        verticalSlug\n        citySlugTitle\n        seoTextHtml\n        debugData\n      }\n    }","variables":{"ufsSessionId":"4b8697cc5b2056f4f67288f7bde39ad2","searchMethod":"direct","locale":"de","q":"verpackungen","filters":[{"name":"Mitarbeiteranzahl","selected":["200+","50-199"]}],"page":1,"sort":"ad_engine","tracking":true,"disableCityExtraction":false,"cityExtractionRadius":"50km"}}'
url = 'https://www.wlw.de/unified_search_backend/graphql'

r = requests.post(url, headers=headers, data=json.dumps(payload))
r.content
Out[34]: b'{"errors":[{"message":"No query string was present"}]}'

Answer 1

您的负载已经是一个JSON字符串，因此无需再次对其调用JSON.dumps（）；这是双重编码，因此api会抱怨有效负载

试一试

payload = '{"query":"query(\n        $ufsSessionId: String,\n        $searchMethod: String,\n        $q: String,\n        $conceptSlug: String,\n        $citySlug: String,\n        $filters: [FilterInput!],\n        $page: Int,\n        $sort: String,\n        $tracking: Boolean,\n        $disableCityExtraction: Boolean,\n        $cityExtractionRadius: String\n      ) {\n      search(\n        ufsSessionId: $ufsSessionId,\n        searchMethod: $searchMethod,\n        q: $q,\n        conceptSlug: $conceptSlug,\n        citySlug: $citySlug,\n        filters: $filters,\n        page: $page,\n        sort: $sort,\n        tracking: $tracking,\n        disableCityExtraction: $disableCityExtraction,\n        cityExtractionRadius: $cityExtractionRadius\n      ) {\n        ufsSessionId\n        paging {\n          total, currentPage, totalPages\n        }\n        companies {\n          id, uuid, name, highlightedName, description, highlightedDescription, secondaryHighlightedDescription,\n          homepage, zipcode, city, countryCode, certificates, logoPath, profilePagePath, distance, email,\n          debugData, employeeCount, foundingYear, acronym, customerId, isCustomer, fpaket, spaceType\n          pictures {\n            caption, path, position, sizes\n          }\n          logo {\n            sizes\n          }\n          filterResults {\n            key, title, value, matches\n          }\n          matchingCompanyCategory {\n            id, kpaket, toprankingPosition\n          }\n        }\n        filters {\n          key, name, static, title, type, range {\n            max, min, unit, disabled\n          }\n          values {\n            count, name, title\n          }\n          location {\n            kind\n            defaultRadiuses\n            availableRadiuses {\n              name, count\n            }\n          }\n        }\n        cities {\n          slug, title, count\n        }\n        suggestions {\n          text, score\n        }\n        extractedCity {\n          name, radius\n        }\n        toprankingPositions {\n          companyId, position\n        }\n        companyCategory {\n          id, name\n        }\n        concept {\n          title\n        }\n        recommendations {\n          text, score, slug\n        }\n        canonicalConceptSlug\n        canonicalCompanyCategorySlug\n        conceptSlugTitle\n        companyCategorySlugTitle\n        verticalSlug\n        citySlugTitle\n        seoTextHtml\n        debugData\n      }\n    }","variables":{"ufsSessionId":"4b8697cc5b2056f4f67288f7bde39ad2","searchMethod":"direct","locale":"de","q":"verpackungen","filters":[{"name":"Mitarbeiteranzahl","selected":["200+","50-199"]}],"page":1,"sort":"ad_engine","tracking":true,"disableCityExtraction":false,"cityExtractionRadius":"50km"}}'
r = requests.post(url, headers=headers, data=payload)

要了解双重编码的含义，请比较两个有效负载：

print(repr(payload))
print(repr(json.dumps(payload)))

Answer 2

负载是个问题，也许你可以通过添加url来解决它

+'?query=the_item_of_query_in_your_payload_dict'

并删除有效载荷dict