feat: add `robots.txt` setting (close #4303)

internal/bootstrap/data/setting.go

@@ -86,6 +86,7 @@ func InitialSettings() []model.SettingItem {
 		{Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE},
 		{Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE},
 		{Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE},
+		{Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE},
 		// style settings
 		{Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE},
 		{Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE},

internal/conf/const.go

@@ -15,6 +15,7 @@ const (
 	Announcement = "announcement"
 	AllowIndexed = "allow_indexed"
 	AllowMounted = "allow_mounted"
+	RobotsTxt    = "robots_txt"
 
 	Logo      = "logo"
 	Favicon   = "favicon"

server/handles/helper.go

@@ -16,6 +16,10 @@ func Favicon(c *gin.Context) {
 	c.Redirect(302, setting.GetStr(conf.Favicon))
 }
 
+func Robots(c *gin.Context) {
+	c.String(200, setting.GetStr(conf.RobotsTxt))
+}
+
 func Plist(c *gin.Context) {
 	linkNameB64 := strings.TrimSuffix(c.Param("link_name"), ".plist")
 	linkName, err := utils.SafeAtob(linkNameB64)

server/router.go

@@ -24,6 +24,9 @@ func Init(e *gin.Engine) {
 	g.Any("/ping", func(c *gin.Context) {
 		c.String(200, "pong")
 	})
+	g.GET("/favicon.ico", handles.Favicon)
+	g.GET("/robots.txt", handles.Robots)
+	g.GET("/i/:link_name", handles.Plist)
 	common.SecretKey = []byte(conf.Conf.JwtSecret)
 	g.Use(middlewares.StoragesLoaded)
 	if conf.Conf.MaxConnections > 0 {
@@ -31,8 +34,6 @@ func Init(e *gin.Engine) {
 	}
 	WebDav(g.Group("/dav"))
 
-	g.GET("/favicon.ico", handles.Favicon)
-	g.GET("/i/:link_name", handles.Plist)
 	g.GET("/d/*path", middlewares.Down, handles.Down)
 	g.GET("/p/*path", middlewares.Down, handles.Proxy)