perf(GeoIPMatcher): faster heuristic matching with reduced memory usage (#5289)

app/dns/nameserver.go

@@ -29,8 +29,8 @@ type Client struct {
 	server        Server
 	skipFallback  bool
 	domains       []string
-	expectedIPs   []*router.GeoIPMatcher
-	unexpectedIPs []*router.GeoIPMatcher
+	expectedIPs   router.GeoIPMatcher
+	unexpectedIPs router.GeoIPMatcher
 	actPrior      bool
 	actUnprior    bool
 	tag           string
@@ -154,23 +154,21 @@ func NewClient(
 		}
 
 		// Establish expected IPs
-		var expectedMatchers []*router.GeoIPMatcher
-		for _, geoip := range ns.ExpectedGeoip {
-			matcher, err := router.GlobalGeoIPContainer.Add(geoip)
+		var expectedMatcher router.GeoIPMatcher
+		if len(ns.ExpectedGeoip) > 0 {
+			expectedMatcher, err = router.BuildOptimizedGeoIPMatcher(ns.ExpectedGeoip...)
 			if err != nil {
 				return errors.New("failed to create expected ip matcher").Base(err).AtWarning()
 			}
-			expectedMatchers = append(expectedMatchers, matcher)
 		}
 
 		// Establish unexpected IPs
-		var unexpectedMatchers []*router.GeoIPMatcher
-		for _, geoip := range ns.UnexpectedGeoip {
-			matcher, err := router.GlobalGeoIPContainer.Add(geoip)
+		var unexpectedMatcher router.GeoIPMatcher
+		if len(ns.UnexpectedGeoip) > 0 {
+			unexpectedMatcher, err = router.BuildOptimizedGeoIPMatcher(ns.UnexpectedGeoip...)
 			if err != nil {
 				return errors.New("failed to create unexpected ip matcher").Base(err).AtWarning()
 			}
-			unexpectedMatchers = append(unexpectedMatchers, matcher)
 		}
 
 		if len(clientIP) > 0 {
@@ -192,8 +190,8 @@ func NewClient(
 		client.server = server
 		client.skipFallback = ns.SkipFallback
 		client.domains = rules
-		client.expectedIPs = expectedMatchers
-		client.unexpectedIPs = unexpectedMatchers
+		client.expectedIPs = expectedMatcher
+		client.unexpectedIPs = unexpectedMatcher
 		client.actPrior = ns.ActPrior
 		client.actUnprior = ns.ActUnprior
 		client.tag = tag
@@ -243,32 +241,32 @@ func (c *Client) QueryIP(ctx context.Context, domain string, option dns.IPOption
 		return nil, 0, dns.ErrEmptyResponse
 	}
 
-	if len(c.expectedIPs) > 0 && !c.actPrior {
-		ips = router.MatchIPs(c.expectedIPs, ips, false)
+	if c.expectedIPs != nil && !c.actPrior {
+		ips, _ = c.expectedIPs.FilterIPs(ips)
 		errors.LogDebug(context.Background(), "domain ", domain, " expectedIPs ", ips, " matched at server ", c.Name())
 		if len(ips) == 0 {
 			return nil, 0, dns.ErrEmptyResponse
 		}
 	}
 
-	if len(c.unexpectedIPs) > 0 && !c.actUnprior {
-		ips = router.MatchIPs(c.unexpectedIPs, ips, true)
+	if c.unexpectedIPs != nil && !c.actUnprior {
+		_, ips = c.unexpectedIPs.FilterIPs(ips)
 		errors.LogDebug(context.Background(), "domain ", domain, " unexpectedIPs ", ips, " matched at server ", c.Name())
 		if len(ips) == 0 {
 			return nil, 0, dns.ErrEmptyResponse
 		}
 	}
 
-	if len(c.expectedIPs) > 0 && c.actPrior {
-		ipsNew := router.MatchIPs(c.expectedIPs, ips, false)
+	if c.expectedIPs != nil && c.actPrior {
+		ipsNew, _ := c.expectedIPs.FilterIPs(ips)
 		if len(ipsNew) > 0 {
 			ips = ipsNew
 			errors.LogDebug(context.Background(), "domain ", domain, " priorIPs ", ips, " matched at server ", c.Name())
 		}
 	}
 
-	if len(c.unexpectedIPs) > 0 && c.actUnprior {
-		ipsNew := router.MatchIPs(c.unexpectedIPs, ips, true)
+	if c.unexpectedIPs != nil && c.actUnprior {
+		_, ipsNew := c.unexpectedIPs.FilterIPs(ips)
 		if len(ipsNew) > 0 {
 			ips = ipsNew
 			errors.LogDebug(context.Background(), "domain ", domain, " unpriorIPs ", ips, " matched at server ", c.Name())

app/router/condition.go

@@ -96,61 +96,53 @@ func (m *DomainMatcher) Apply(ctx routing.Context) bool {
 	return m.ApplyDomain(domain)
 }
 
-type MultiGeoIPMatcher struct {
-	matchers []*GeoIPMatcher
-	asType   string // local, source, target
+type MatcherAsType byte
+
+const (
+	MatcherAsType_Local MatcherAsType = iota
+	MatcherAsType_Source
+	MatcherAsType_Target
+	MatcherAsType_VlessRoute // for port
+)
+
+type IPMatcher struct {
+	matcher GeoIPMatcher
+	asType  MatcherAsType
 }
 
-func NewMultiGeoIPMatcher(geoips []*GeoIP, asType string) (*MultiGeoIPMatcher, error) {
-	var matchers []*GeoIPMatcher
-	for _, geoip := range geoips {
-		matcher, err := GlobalGeoIPContainer.Add(geoip)
-		if err != nil {
-			return nil, err
-		}
-		matchers = append(matchers, matcher)
+func NewIPMatcher(geoips []*GeoIP, asType MatcherAsType) (*IPMatcher, error) {
+	matcher, err := BuildOptimizedGeoIPMatcher(geoips...)
+	if err != nil {
+		return nil, err
 	}
-
-	matcher := &MultiGeoIPMatcher{
-		matchers: matchers,
-		asType:   asType,
-	}
-
-	return matcher, nil
+	return &IPMatcher{matcher: matcher, asType: asType}, nil
 }
 
 // Apply implements Condition.
-func (m *MultiGeoIPMatcher) Apply(ctx routing.Context) bool {
+func (m *IPMatcher) Apply(ctx routing.Context) bool {
 	var ips []net.IP
 
 	switch m.asType {
-	case "local":
+	case MatcherAsType_Local:
 		ips = ctx.GetLocalIPs()
-	case "source":
+	case MatcherAsType_Source:
 		ips = ctx.GetSourceIPs()
-	case "target":
+	case MatcherAsType_Target:
 		ips = ctx.GetTargetIPs()
 	default:
-		panic("unreachable, asType should be local or source or target")
+		panic("unk asType")
 	}
 
-	for _, ip := range ips {
-		for _, matcher := range m.matchers {
-			if matcher.Match(ip) {
-				return true
-			}
-		}
-	}
-	return false
+	return m.matcher.AnyMatch(ips)
 }
 
 type PortMatcher struct {
 	port   net.MemoryPortList
-	asType string // local, source, target
+	asType MatcherAsType
 }
 
 // NewPortMatcher create a new port matcher that can match source or local or destination port
-func NewPortMatcher(list *net.PortList, asType string) *PortMatcher {
+func NewPortMatcher(list *net.PortList, asType MatcherAsType) *PortMatcher {
 	return &PortMatcher{
 		port:   net.PortListFromProto(list),
 		asType: asType,
@@ -160,18 +152,17 @@ func NewPortMatcher(list *net.PortList, asType string) *PortMatcher {
 // Apply implements Condition.
 func (v *PortMatcher) Apply(ctx routing.Context) bool {
 	switch v.asType {
-	case "local":
+	case MatcherAsType_Local:
 		return v.port.Contains(ctx.GetLocalPort())
-	case "source":
+	case MatcherAsType_Source:
 		return v.port.Contains(ctx.GetSourcePort())
-	case "target":
+	case MatcherAsType_Target:
 		return v.port.Contains(ctx.GetTargetPort())
-	case "vlessRoute":
+	case MatcherAsType_VlessRoute:
 		return v.port.Contains(ctx.GetVlessRoute())
 	default:
-		panic("unreachable, asType should be local or source or target")
+		panic("unk asType")
 	}
-
 }
 
 type NetworkMatcher struct {

app/router/condition_geoip_test.go

@@ -35,33 +35,6 @@ func getAssetPath(file string) (string, error) {
 	return path, nil
 }
 
-func TestGeoIPMatcherContainer(t *testing.T) {
-	container := &router.GeoIPMatcherContainer{}
-
-	m1, err := container.Add(&router.GeoIP{
-		CountryCode: "CN",
-	})
-	common.Must(err)
-
-	m2, err := container.Add(&router.GeoIP{
-		CountryCode: "US",
-	})
-	common.Must(err)
-
-	m3, err := container.Add(&router.GeoIP{
-		CountryCode: "CN",
-	})
-	common.Must(err)
-
-	if m1 != m3 {
-		t.Error("expect same matcher for same geoip, but not")
-	}
-
-	if m1 == m2 {
-		t.Error("expect different matcher for different geoip, but actually same")
-	}
-}
-
 func TestGeoIPMatcher(t *testing.T) {
 	cidrList := []*router.CIDR{
 		{Ip: []byte{0, 0, 0, 0}, Prefix: 8},
@@ -80,8 +53,10 @@ func TestGeoIPMatcher(t *testing.T) {
 		{Ip: []byte{91, 108, 4, 0}, Prefix: 16},
 	}
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(cidrList))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: cidrList,
+	})
+	common.Must(err)
 
 	testCases := []struct {
 		Input  string
@@ -140,8 +115,10 @@ func TestGeoIPMatcherRegression(t *testing.T) {
 		{Ip: []byte{98, 108, 20, 0}, Prefix: 23},
 	}
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(cidrList))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: cidrList,
+	})
+	common.Must(err)
 
 	testCases := []struct {
 		Input  string
@@ -171,9 +148,11 @@ func TestGeoIPReverseMatcher(t *testing.T) {
 		{Ip: []byte{8, 8, 8, 8}, Prefix: 32},
 		{Ip: []byte{91, 108, 4, 0}, Prefix: 16},
 	}
-	matcher := &router.GeoIPMatcher{}
-	matcher.SetReverseMatch(true) // Reverse match
-	common.Must(matcher.Init(cidrList))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: cidrList,
+	})
+	common.Must(err)
+	matcher.SetReverse(true) // Reverse match
 
 	testCases := []struct {
 		Input  string
@@ -206,8 +185,10 @@ func TestGeoIPMatcher4CN(t *testing.T) {
 	ips, err := loadGeoIP("CN")
 	common.Must(err)
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(ips))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
+	common.Must(err)
 
 	if matcher.Match([]byte{8, 8, 8, 8}) {
 		t.Error("expect CN geoip doesn't contain 8.8.8.8, but actually does")
@@ -218,8 +199,10 @@ func TestGeoIPMatcher6US(t *testing.T) {
 	ips, err := loadGeoIP("US")
 	common.Must(err)
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(ips))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
+	common.Must(err)
 
 	if !matcher.Match(net.ParseAddress("2001:4860:4860::8888").IP()) {
 		t.Error("expect US geoip contain 2001:4860:4860::8888, but actually not")
@@ -254,8 +237,10 @@ func BenchmarkGeoIPMatcher4CN(b *testing.B) {
 	ips, err := loadGeoIP("CN")
 	common.Must(err)
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(ips))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
+	common.Must(err)
 
 	b.ResetTimer()
 
@@ -268,8 +253,10 @@ func BenchmarkGeoIPMatcher6US(b *testing.B) {
 	ips, err := loadGeoIP("US")
 	common.Must(err)
 
-	matcher := &router.GeoIPMatcher{}
-	common.Must(matcher.Init(ips))
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
+	common.Must(err)
 
 	b.ResetTimer()
 

app/router/condition_test.go

@@ -447,7 +447,7 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 		})
 	}
 
-	matcher, err := NewMultiGeoIPMatcher(geoips, "target")
+	matcher, err := NewIPMatcher(geoips, MatcherAsType_Target)
 	common.Must(err)
 
 	ctx := withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.8.8"), 80)})

app/router/config.go

@@ -46,7 +46,7 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if rr.VlessRouteList != nil {
-		conds.Add(NewPortMatcher(rr.VlessRouteList, "vlessRoute"))
+		conds.Add(NewPortMatcher(rr.VlessRouteList, MatcherAsType_VlessRoute))
 	}
 
 	if len(rr.InboundTag) > 0 {
@@ -54,15 +54,15 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if rr.PortList != nil {
-		conds.Add(NewPortMatcher(rr.PortList, "target"))
+		conds.Add(NewPortMatcher(rr.PortList, MatcherAsType_Target))
 	}
 
 	if rr.SourcePortList != nil {
-		conds.Add(NewPortMatcher(rr.SourcePortList, "source"))
+		conds.Add(NewPortMatcher(rr.SourcePortList, MatcherAsType_Source))
 	}
 
 	if rr.LocalPortList != nil {
-		conds.Add(NewPortMatcher(rr.LocalPortList, "local"))
+		conds.Add(NewPortMatcher(rr.LocalPortList, MatcherAsType_Local))
 	}
 
 	if len(rr.Networks) > 0 {
@@ -70,7 +70,7 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if len(rr.Geoip) > 0 {
-		cond, err := NewMultiGeoIPMatcher(rr.Geoip, "target")
+		cond, err := NewIPMatcher(rr.Geoip, MatcherAsType_Target)
 		if err != nil {
 			return nil, err
 		}
@@ -78,7 +78,7 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if len(rr.SourceGeoip) > 0 {
-		cond, err := NewMultiGeoIPMatcher(rr.SourceGeoip, "source")
+		cond, err := NewIPMatcher(rr.SourceGeoip, MatcherAsType_Source)
 		if err != nil {
 			return nil, err
 		}
@@ -86,7 +86,7 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if len(rr.LocalGeoip) > 0 {
-		cond, err := NewMultiGeoIPMatcher(rr.LocalGeoip, "local")
+		cond, err := NewIPMatcher(rr.LocalGeoip, MatcherAsType_Local)
 		if err != nil {
 			return nil, err
 		}