From 89f78bd4c7e469aa28b31f35d570924afb6fdbb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A4=9C=E6=B3=95=E4=B9=8B=E4=B9=A6=28appotry=29?=
 <andycrusoe@gmail.com>
Date: Mon, 24 Apr 2023 20:08:30 +0800
Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0nginx=E9=98=B2=E6=AD=A2?=
 =?UTF-8?q?=E7=88=AC=E8=99=AB=E7=88=AC=E5=8F=96=E9=85=8D=E7=BD=AE=20(#1187?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* 添加nginx防止爬虫爬取配置

* Update nginx.conf

* Update README.md
---
 README.md                       | 16 ++++++++++++++++
 docker-compose/nginx/nginx.conf |  7 +++++++
 2 files changed, 23 insertions(+)

diff --git a/README.md b/README.md
index 9d811c5..0d96f87 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@
 			- [Docker 参数示例](#docker-参数示例)
 			- [Docker build \& Run](#docker-build--run)
 			- [Docker compose](#docker-compose)
+			- [防止爬虫抓取](#防止爬虫抓取)
 		- [使用 Railway 部署](#使用-railway-部署)
 			- [Railway 环境变量](#railway-环境变量)
 		- [手动打包](#手动打包)
@@ -234,6 +235,21 @@ services:
 ```
 - `OPENAI_API_BASE_URL`  可选，设置 `OPENAI_API_KEY` 时可用
 - `OPENAI_API_MODEL`  可选，设置 `OPENAI_API_KEY` 时可用
+
+#### 防止爬虫抓取
+
+**nginx**
+
+将下面配置填入nginx配置文件中，可以参考 `docker-compose/nginx/nginx.conf` 文件中添加反爬虫的方法
+
+```
+    # 防止爬虫抓取
+    if ($http_user_agent ~* "360Spider|JikeSpider|Spider|spider|bot|Bot|2345Explorer|curl|wget|webZIP|qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot|NSPlayer|bingbot")
+    {
+      return 403;
+    }
+```
+
 ###  使用 Railway 部署
 
 [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/new/template/yytmgc)
diff --git a/docker-compose/nginx/nginx.conf b/docker-compose/nginx/nginx.conf
index 0f9df35..fa46d2b 100644
--- a/docker-compose/nginx/nginx.conf
+++ b/docker-compose/nginx/nginx.conf
@@ -3,6 +3,13 @@ server {
 	server_name  localhost;
 	charset utf-8;
 	error_page   500 502 503 504  /50x.html;
+	
+	# 防止爬虫抓取
+	if ($http_user_agent ~* "360Spider|JikeSpider|Spider|spider|bot|Bot|2345Explorer|curl|wget|webZIP|qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot|NSPlayer|bingbot")
+	{
+		return 403;
+	}
+	
 	location / {
 			root /usr/share/nginx/html;
    		try_files $uri /index.html;