diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 214ba7d75..495254b9d 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -21,6 +21,7 @@ import (
 	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"text/template"
@@ -76,6 +77,7 @@ func Main() int {
 		NotificationHandler: notificationHandler,
 		QueryEngine:         queryEngine,
 		ExternalURL:         cfg.web.ExternalURL,
+		BaseDir:             filepath.Dir(cfg.configFile),
 	})
 
 	flags := map[string]string{}
diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index b144e6219..791fa069f 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -84,6 +84,10 @@ func checkConfig(t cli.Term, filename string) ([]string, error) {
 	}
 	var ruleFiles []string
 	for _, rf := range cfg.RuleFiles {
+		if !filepath.IsAbs(rf) {
+			rf = filepath.Join(filepath.Dir(filename), rf)
+		}
+
 		rfs, err := filepath.Glob(rf)
 		if err != nil {
 			return nil, err
diff --git a/rules/manager.go b/rules/manager.go
index 3f31b712d..c862fe677 100644
--- a/rules/manager.go
+++ b/rules/manager.go
@@ -104,6 +104,7 @@ type Manager struct {
 	notificationHandler *notification.NotificationHandler
 
 	externalURL *url.URL
+	baseDir     string
 }
 
 // ManagerOptions bundles options for the Manager.
@@ -115,6 +116,7 @@ type ManagerOptions struct {
 	SampleAppender      storage.SampleAppender
 
 	ExternalURL *url.URL
+	BaseDir     string
 }
 
 // NewManager returns an implementation of Manager, ready to be started
@@ -129,6 +131,7 @@ func NewManager(o *ManagerOptions) *Manager {
 		queryEngine:         o.QueryEngine,
 		notificationHandler: o.NotificationHandler,
 		externalURL:         o.ExternalURL,
+		baseDir:             o.BaseDir,
 	}
 	return manager
 }
@@ -327,6 +330,10 @@ func (m *Manager) ApplyConfig(conf *config.Config) bool {
 
 	var files []string
 	for _, pat := range conf.RuleFiles {
+		if !filepath.IsAbs(pat) {
+			pat = filepath.Join(m.baseDir, pat)
+		}
+
 		fs, err := filepath.Glob(pat)
 		if err != nil {
 			// The only error can be a bad pattern.
@@ -356,6 +363,7 @@ func (m *Manager) loadRuleFiles(filenames ...string) error {
 		if err != nil {
 			return fmt.Errorf("error parsing %s: %s", fn, err)
 		}
+
 		for _, stmt := range stmts {
 			switch r := stmt.(type) {
 			case *promql.AlertStmt: