[COMMITTED] Add more robots.txt rules, block Huawei Cloud and bingbot

[Mark Wielaard]

---
 config-files/etc/apache2/block.include     | 30 +++++++++++++++++++++-
 config-files/srv/forgejo/custom/robots.txt |  2 ++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/config-files/etc/apache2/block.include b/config-files/etc/apache2/block.include
index 5fd34c77d0ab..644f3f22c233 100644
--- a/config-files/etc/apache2/block.include
+++ b/config-files/etc/apache2/block.include
@@ -7,9 +7,37 @@ Require not ip 47.80.0.0/13
 Require not ip 47.76.0.0/14
 Require not ip 47.74.0.0/15
 
+# Huawei-Cloud-SG
+Require not ip 111.119.192.0/18
+Require not ip 124.243.128.0/18
+Require not ip 166.108.192.0/19
+# Huawei-HK-CLOUDS
+Require not ip 159.138.0.0/17
+Require not ip 159.138.128.0/19
+# More Huawei Cloud
+Require not ip 101.44.16.0/20
+Require not ip 101.44.32.0/19
+Require not ip 101.44.64.0/18
+Require not ip 101.44.128.0/18
+# Huawei Cloud Singapore POP
+Require not ip 190.92.192.0/19
+# Huawei Cloud Hong Kong POP
+Require not ip 190.92.224.0/19
+# Huawei Clouds HongKong
+# Huawei Clouds Argentina
+# Huawei Clouds Peru
+Require not ip 119.8.24.0/21
+Require not ip 119.8.32.0/19
+Require not ip 119.8.64.0/18
+Require not ip 119.8.128.0/18
+Require not ip 119.8.192.0/19
+Require not ip 119.8.224.0/20
+# Huawei Cloud SG POP
+Require not ip 101.46.0.0/20
+
 # Bad bots ignoring robots.txt
 Require not expr %{HTTP_USER_AGENT} =~ m/DataForSeoBot/
+Require not expr %{HTTP_USER_AGENT} =~ m/bingbot/
 
 </RequireAll>
 </Location>
-
diff --git a/config-files/srv/forgejo/custom/robots.txt b/config-files/srv/forgejo/custom/robots.txt
index 16c24385451e..ecf1179457c0 100644
--- a/config-files/srv/forgejo/custom/robots.txt
+++ b/config-files/srv/forgejo/custom/robots.txt
@@ -4,7 +4,9 @@ Disallow: /avatars
 Disallow: /user/*
 Disallow: /*/*/src/branch/*
 Disallow: /*/*/src/commit/*
+Disallow: /*/*/src/tag/*
 Disallow: /*/*/commit/*
+Disallow: /*/*/rss/*
 Disallow: /*/*/*/refs/*
 Disallow: /*/*/*/star
 Disallow: /*/*/*/watch
-- 
2.49.0