使用Python和Google NLP API进行网站分类¶

    import cloudscraper
    from bs4 import BeautifulSoup

    scraper = cloudscraper.create_scraper() 
    headers = {'user-agent': 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'}

    try:
        r = scraper.get(<your website>, headers = headers)

        soup = BeautifulSoup(r.text, 'html.parser')
        title = soup.find('title').text
        description = soup.find('meta', attrs={'name': 'description'})

        if "content" in str(description):
            description = description.get("content")
        else:
            description = ""


        h1 = soup.find_all('h1')
        h1_all = ""
        for x in range (len(h1)):
            if x ==  len(h1) -1:
                h1_all = h1_all + h1[x].text
            else:
                h1_all = h1_all + h1[x].text + ". "


        paragraphs_all = ""
        paragraphs = soup.find_all('p')
        for x in range (len(paragraphs)):
            if x ==  len(paragraphs) -1:
                paragraphs_all = paragraphs_all + paragraphs[x].text
            else:
                paragraphs_all = paragraphs_all + paragraphs[x].text + ". "
        h2 = soup.find_all('h2')
        h2_all = ""
        for x in range (len(h2)):
            if x ==  len(h2) -1:
                h2_all = h2_all + h2[x].text
            else:
                h2_all = h2_all + h2[x].text + ". "
        h3 = soup.find_all('h3')
        h3_all = ""
        for x in range (len(h3)):
            if x ==  len(h3) -1:
                h3_all = h3_all + h3[x].text
            else:
                h3_all = h3_all + h3[x].text + ". "

        allthecontent = ""
        allthecontent = str(title) + " " + str(description) + " " + str(h1_all) + " " + str(h2_all) + " " + str(h3_all) + " " + str(paragraphs_all)
        allthecontent = str(allthecontent)[0:999]

    except Exception as e:
            print(e)

    from googletrans import Translator
    translator = Translator()

    try:
            translation = translator.translate(allthecontent).text
            translation = str(translation)[0:999]
            time.sleep(10)

        except Exception as e:
            print(e)

 ```

## 3.- 使用Google NLP API对网站进行分类

最后，我们将使用Google NLP API对网站中的翻译文本进行分类。感谢[Greg Bernhardt](https://importsem.com/getting-started-with-google-nlp-api-using-python/)所发表的这篇文章，我对此API有所了解，因此，如果您想进一步了解该API可以提供的所有功能，我建议您阅读本文。在开始网站分类之前，请务必在[Google Cloud](https://cloud.google.com/)平台上创建API密钥。

为了对网站进行分类，我们将使用以下代码：

```python   
    import os
    from google.cloud import language_v1
    from google.cloud.language_v1 import enums
    from google.cloud import language
    from google.cloud.language import types

    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = <path to your credentials file>

    try:
            text_content = str(translation)
            text_content = text_content[0:1000]

            client = language_v1.LanguageServiceClient()

            type_ = enums.Document.Type.PLAIN_TEXT
            language = "en"
            document = {"content": text_content, "type": type_, "language": language}

            encoding_type = enums.EncodingType.UTF8

            response = client.classify_text(document)
            print(response.categories[0].name)
            print(str(int(round(response.categories[0].confidence,3)*100))+"%")


    except Exception as e:
        print(e)