Add header to get content type

2020-11-03 20:27:43 +00:00
parent c4c3f448b1
commit 410499518e
1 changed files with 5 additions and 2 deletions
@@ -64,8 +64,11 @@ def get_article(url):

 def get_content_type(url):
    try:
-        headers = {'User-Agent': 'Twitterbot/1.0'}
-        return requests.get(url, headers=headers, timeout=2).headers['content-type']
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+            'X-Forwarded-For': '66.249.66.1',
+        }
+        return requests.get(url, headers=headers, timeout=5).headers['content-type']
    except:
        pass